Pup - the AI's own browser

@@ -321,6 +321,77 @@ function pickSummary() {  // Programmatic install of the pinned CfT build. Resolves to { ok, executablePath, buildId } // or { ok:false, errorCode, error }. Deduped via _installInFlight.+// v2.0.398 — SUPERVISED UNPACK. Child process (killable), stall watchdog on bytes written,+// OS tar first on Windows/macOS (bsdtar reads zip natively, fast, no JS streams), the+// extract-zip worker (src/unpack.js) as the fallback and the Linux path. Each method that+// fails or stalls is cleaned up before the next is tried; the caller verifies the executable.+const UNPACK_STALL_MS = 90000;+const UNPACK_MAX_MS = 15 * 60000;+function dirBytes(dir, depth = 0) {+  let total = 0;+  try {+    for (const e of fs.readdirSync(dir, { withFileTypes: true })) {+      const p = path.join(dir, e.name);+      if (e.isDirectory()) { if (depth < 6) total += dirBytes(p, depth + 1); }+      else if (e.isFile()) { try { total += fs.statSync(p).size; } catch {} }+    }+  } catch {}+  return total;+}+function runUnpackMethod(m, dest, onBytes) {+  return new Promise((resolve, reject) => {+    const { spawn } = require('child_process');+    let child;+    try { child = spawn(m.cmd, m.args, { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, cwd: __dirname }); }+    catch (e) { return reject(e); }+    let stderr = '', done = false;+    const finish = (fn, v) => { if (done) return; done = true; clearInterval(timer); fn(v); };+    child.stderr.on('data', (d) => { stderr += String(d); if (stderr.length > 4000) stderr = stderr.slice(-4000); });+    child.stdout.on('data', () => {});+    child.on('error', (e) => finish(reject, e));+    let lastBytes = -1, lastGrowth = Date.now(); const started = Date.now();+    const timer = setInterval(() => {+      const b = dirBytes(dest);+      if (onBytes) { try { onBytes(b); } catch {} }+      if (b !== lastBytes) { lastBytes = b; lastGrowth = Date.now(); }+      if (Date.now() - lastGrowth > UNPACK_STALL_MS) {+        try { child.kill('SIGKILL'); } catch {}+        finish(reject, new Error(`unpack via ${m.name} STALLED: no bytes written for ${Math.round(UNPACK_STALL_MS / 1000)}s (${b} bytes so far); killed it`));+      } else if (Date.now() - started > UNPACK_MAX_MS) {+        try { child.kill('SIGKILL'); } catch {}+        finish(reject, new Error(`unpack via ${m.name} exceeded ${Math.round(UNPACK_MAX_MS / 60000)} min; killed it`));+      }+    }, 5000);+    child.on('exit', (code, sig) => {+      const b = dirBytes(dest); if (onBytes) { try { onBytes(b); } catch {} }+      if (code === 0) finish(resolve, { bytes: b });+      else finish(reject, new Error(`${m.name} exited ${code}${sig ? ' (' + sig + ')' : ''}: ${stderr.trim().slice(-300)}`));+    });+  });+}+async function unpackSupervised(archive, dest, { onBytes, onMethod } = {}) {+  const methods = [];+  const tar = { name: 'tar', cmd: 'tar', args: ['-xf', archive, '-C', dest] };+  const worker = { name: 'extract-zip', cmd: process.execPath, args: [path.join(__dirname, 'unpack.js'), archive, dest] };+  if (process.platform === 'linux') methods.push(worker, tar); else methods.push(tar, worker);+  let lastErr = null;+  for (const m of methods) {+    try { fs.rmSync(dest, { recursive: true, force: true }); } catch {}+    fs.mkdirSync(dest, { recursive: true });+    if (onMethod) { try { onMethod(m.name); } catch {} }+    const t0 = Date.now();+    try {+      const r = await runUnpackMethod(m, dest, onBytes);+      return { method: m.name, bytes: r.bytes, ms: Date.now() - t0 };+    } catch (e) {+      lastErr = e;+      console.log(`[chrome] ${e.message} — ${methods.indexOf(m) < methods.length - 1 ? 'trying the next unpack method' : 'no unpack method left'}`);+    }+  }+  try { fs.rmSync(dest, { recursive: true, force: true }); } catch {}+  throw lastErr || new Error('no unpack method available');+}+ function installChrome({ onProgress } = {}) {   if (_installInFlight) return _installInFlight; @@ -395,10 +466,18 @@ function installChrome({ onProgress } = {}) {       } catch (e) { console.log(`[chrome] half-populated-cache check skipped: ${e.message}`); }       console.log(`[chrome] installing Chrome for Testing ${buildId} (${platform}) into ${cacheDir()} (disk free: ${freeMb == null ? 'unknown' : freeMb + ' MB'}) ...`); -      const installed = await install({+      // v2.0.398: DOWNLOAD ONLY here. puppeteer's own unpack (extract-zip, in-process) hung forever+      // on arav-rog at the third file of a valid, fully downloaded archive, with nothing to kill+      // and nothing reported: that is how the half-populated folders that break first runs are+      // born. pup unpacks in a SUPERVISED CHILD PROCESS (OS tar first, extract-zip worker as+      // fallback) into a .partial folder with a stall watchdog, verifies the executable, then+      // renames the folder into place atomically. A failure leaves nothing behind but a clear+      // lastError, and the next attempt starts clean.+      const archiveRaw = await install({         browser: Browser.CHROME,         buildId,         cacheDir: cacheDir(),+        unpack: false,         downloadProgressCallback: (downloaded, total) => {           state.bytesDownloaded = downloaded;           state.bytesTotal = total;@@ -406,8 +485,30 @@ function installChrome({ onProgress } = {}) {           if (onProgress) { try { onProgress(state.progressPct, downloaded, total); } catch {} }         },       });--      const ep = (installed && installed.executablePath) || detectChrome().executablePath;+      const archive = typeof archiveRaw === 'string' ? archiveRaw : (archiveRaw && (archiveRaw.path || archiveRaw.executablePath)) || null;+      if (!archive || !fileExists(archive)) {+        state.errorCode = 'chrome_download_failed';+        throw new Error(`the Chrome for Testing archive is not on disk after the download (${archive || 'no path returned'})`);+      }+      const { computeExecutablePath: _cep } = require('@puppeteer/browsers');+      const expectedExe = _cep({ browser: Browser.CHROME, buildId, cacheDir: cacheDir() });+      let versionedDir = path.dirname(expectedExe);+      while (versionedDir && !path.basename(versionedDir).includes(buildId) && path.dirname(versionedDir) !== versionedDir) versionedDir = path.dirname(versionedDir);+      const partial = versionedDir + '.partial';+      state.phase = 'unpacking'; state.unpackStartedAt = Date.now(); state.unpackBytes = 0; state.unpackMethod = null;+      console.log(`[chrome] downloaded ${archive} (${state.bytesTotal} bytes) — unpacking into ${partial}`);+      const un = await unpackSupervised(archive, partial, { onBytes: (b) => { state.unpackBytes = b; }, onMethod: (m) => { state.unpackMethod = m; } });+      const exeInPartial = path.join(partial, path.relative(versionedDir, expectedExe));+      if (!fileExists(exeInPartial)) {+        state.errorCode = 'chrome_unpack_incomplete';+        try { fs.rmSync(partial, { recursive: true, force: true }); } catch {}+        throw new Error(`unpack via ${un.method} finished but ${exeInPartial} is missing`);+      }+      try { if (fs.existsSync(versionedDir)) fs.rmSync(versionedDir, { recursive: true, force: true }); } catch {}+      fs.renameSync(partial, versionedDir);+      try { fs.unlinkSync(archive); } catch {}+      console.log(`[chrome] unpacked via ${un.method} in ${Math.round((Date.now() - state.unpackStartedAt) / 1000)}s (${state.unpackBytes} bytes) -> ${versionedDir}`);+      const ep = expectedExe;       if (!fileExists(ep)) {         state.phase = 'failed';         state.errorCode = 'chrome_install_incomplete';@@ -430,6 +531,7 @@ function installChrome({ onProgress } = {}) {         if (/ENOSPC|no space left|not enough space/i.test(msg)) state.errorCode = 'chrome_install_no_disk';         else if (/EACCES|EPERM|permission/i.test(msg))          state.errorCode = 'chrome_install_permission';         else if (/ENOTFOUND|ETIMEDOUT|ECONNRESET|network|getaddrinfo|proxy|tunneling/i.test(msg)) state.errorCode = 'chrome_download_network';+        else if (/unpack|STALLED|exited \d/i.test(msg))          state.errorCode = 'chrome_unpack_failed';         else state.errorCode = 'chrome_download_failed';       }       state.error = state.errorCode === 'chrome_install_no_disk'@@ -486,7 +588,7 @@ async function ensureChromeReady({ background = true, onProgress } = {}) { function readiness() {   const b = resolveBrowser();   const det = detectChrome();-  const installing = !!_installInFlight || state.phase === 'installing';+  const installing = !!_installInFlight || state.phase === 'installing' || state.phase === 'unpacking';   // v1.9.94 (issue #22 Finding 4) — `ready` means "a drivable browser EXISTS", nothing more. The old   // `&& !installing` conflated "a background CfT download is in flight" with "nothing is drivable", so   // an in-flight (or STALLED) last-resort download reported the whole subsystem not-ready even on a box@@ -513,6 +615,11 @@ function readiness() {     installing,     installPhase: state.phase,     installProgressPct: state.progressPct,+    // v2.0.398: pup owns the unpack (supervised child process); bytes written so far and the+    // ratio to the archive (a CfT zip unpacks to ~2.3x its size) give a real percent.+    unpackBytes: state.unpackBytes || 0,+    unpackElapsedSec: (state.phase === 'unpacking' && state.unpackStartedAt) ? Math.round((Date.now() - state.unpackStartedAt) / 1000) : null,+    unpackMethod: state.unpackMethod || null,     bytesDownloaded: state.bytesDownloaded,     bytesTotal: state.bytesTotal,     // v2.0.383 (#59 measured progress): elapsed + ETA from the actual byte rate, never a constant.@@ -1047,6 +1154,7 @@ function pruneBrokenCacheDirs({ keepBuildId = null } = {}) {       let st = null; try { st = fs.statSync(dir); } catch (e) { continue; }       if (!st.isDirectory()) continue;       if (keepBuildId && name.includes(keepBuildId)) continue;   // the install path handles its own build+      if (name.endsWith('.partial')) { try { if (Date.now() - fs.statSync(dir).mtimeMs < 30 * 60000) continue; } catch (e) {} }   // a live unpack in progress       if (state.installing && state.buildId && name.includes(state.buildId)) continue;   // being written right now       if (hasCftExecutable(dir)) continue;       try {@@ -1096,7 +1204,7 @@ function warmup() {       console.log(`[warmup] no native browser + CfT missing + low disk (${free}MB) — NOT auto-fetching; readiness reports lowDisk and opens will surface the disk hint`);       return;     }-    console.log('[warmup] no installed Chromium on this box — background-fetching Chrome for Testing as the last-resort browser');+    if (!hasNative) console.log('[warmup] no installed Chromium on this box — background-fetching Chrome for Testing as the last-resort browser');     ensureChromeReady({ background: true });   } catch (e) { console.error(`[warmup] ${e && e.message}`); } }@@ -1107,5 +1215,6 @@ module.exports = {   setDefaultBrowser, readDefault, clearDefault, markVerified, pickSummary,   installChromeStable, chromeStableStatus, warmup, setPreferCft, expireTemporaryDefault,   pruneBrokenCacheDirs, hasCftExecutable,+  unpackSupervised, dirBytes,   // v2.0.398: exported for tests/check_unpack.js   abCommand, // v1.8.40: reused by server.js to call AD verbs (e.g. desktop_taskbar) via the direct-API };