← Commit history

Ship 1.9.46: crash-sentinel recovery breaker and deferred AUMID prune

John Lauer ·1f03442d63 · ·parent ca626a6
4 files changed +47−13
adom-bridge-puppeteer-manifest.json+4−4
@@ -1,10 +1,10 @@ {   "manifest_version": 1,   "name": "puppeteer",-  "version": "1.9.45",-  "url": "https://wiki.adom.inc/download/adom/adom-desktop-puppeteer-bridge/1.9.45/adom-bridge-puppeteer-v1.9.45.zip",-  "sha256": "ced15249096d2a68d2d8db3485195f344aecb159c7ce7d2dd53892a1c0b93d67",-  "size": 293875,+  "version": "1.9.46",+  "url": "https://wiki.adom.inc/download/adom/adom-desktop-puppeteer-bridge/1.9.46/adom-bridge-puppeteer-v1.9.46.zip",+  "sha256": "22ce18f95b9f1c881659574bf29d4ba7968e4b6e442b04a48afdd35acbcccf15",+  "size": 294696,   "verbPrefixes": [     "browser_",     "desktop_recorder_",
src/BRIDGE_VERSION+1−1
@@ -1 +1 @@-1.9.45+1.9.46
src/bridge.json+1−1
@@ -2,7 +2,7 @@   "manifest_version": 1,   "name": "puppeteer",   "displayName": "Puppeteer (Chrome for Testing + Native)",-  "version": "1.9.45",+  "version": "1.9.46",   "description": "Drives a browser ALREADY on the machine \u2014 installed Chrome, else Microsoft Edge (every Windows PC ships with Edge) \u2014 launched with a FRESH isolated profile, so the common case has NO ~150 MB Chrome-for-Testing download and it never touches the user's real tabs/logins. Binds its local server to LOOPBACK (127.0.0.1) only, so no Windows Firewall 'allow Node through the firewall' prompt ever pops. Chrome for Testing is fetched only as a LAST RESORT if the machine has no Chromium browser at all. SPAWN-VERIFIED fallback: pup tries each detected browser (Chrome \u2192 Edge \u2192 CfT) and actually launches it, falling through to the next if one won't start (and handling browsers like Edge that relaunch onto another PID), so a corrupt/partial install never hard-fails an open. The browser that launches is cached as the default (persisted across restarts) so it isn't re-probed each time. On spawn the bridge SELF-WARMS: detects browsers + disk, and only if the box has no Chrome/Edge and no cached CfT does it background-fetch CfT so the first open is instant. browser_use pins a specific browser (chrome | edge | cft | auto); {browser:'chrome',install:true} downloads & installs REAL Google Chrome (user-scoped, no admin) on an Edge-only box. browser_readiness (READ-ONLY) reports which browser will be used + the full candidate list + the default + free disk. Every failure returns a specific errorCode + _hint (chrome_install_no_disk, chrome_download_network, chrome_install_permission, chrome_stable_not_detected, \u2026). Launches with FULL browser permissions by default for full debug (strictPermissions:true opts out). Headful automation: open/close windows+tabs, navigate, screenshot, eval JS, record. pup is a GENERIC, non-profile browser only \u2014 every session is a fresh isolated Chrome/Edge process, never the user's real signed-in browser; profile defaults to sessionId. To drive the user's REAL Chrome/Edge (their profile, cookies, SSO), use the Adom browser EXTENSION's nbrowser_* verbs \u2014 a separate bridge, not pup's job.",   "homepage": "https://wiki.adom.inc/adom/adom-desktop",   "author": "Adom Inc.",
src/server.js+41−7
@@ -1060,8 +1060,20 @@ async function rescanAllProfiles({ adoptOrphans = false } = {}) { }  // Recover sessions from disk on startup — reconnect to existing Chrome processes+// v1.9.45: crash sentinel. Written before the risky recovery ops for ONE session and removed+// after that session settles, so a leftover file at boot names exactly the session that killed us.+const RECOVERY_SENTINEL = path.join(SESSIONS_DIR, '.recovery-in-flight');+let _poisonSessionId = null;+ async function recoverSessions() {   const files = listSessionFiles();+  try {+    if (fs.existsSync(RECOVERY_SENTINEL)) {+      _poisonSessionId = fs.readFileSync(RECOVERY_SENTINEL, 'utf8').trim() || null;+      if (_poisonSessionId) console.error(`[recovery] last boot died while recovering "${_poisonSessionId}" — that ONE session will be quarantined`);+      fs.unlinkSync(RECOVERY_SENTINEL);+    }+  } catch (e) {}   if (files.length === 0) return;   console.log(`Recovering ${files.length} session(s) from disk...`); @@ -1082,14 +1094,22 @@ async function recoverSessions() {     // SUCCEEDS clears the marker (saveSessionFile rewrites without it) — so a leftover marker     // means "this file crashed recovery," and one strike quarantines it. Bounds any poison file     // to a single crash instead of an unbounded loop.-    if (Number(sf.recoverAttempts || 0) >= 1) {-      console.error(`Session "${sessionId}" left an INCOMPLETE recovery attempt last boot (likely crashed pup) — QUARANTINING it (deleting the poison file) instead of retrying`);+    // v1.9.45: the per-session `recoverAttempts` strike above was WRONG and cost John all 6 of his+    // windows. It quarantined any file carrying an unfinished-attempt marker, but "unfinished" is+    // not the same as "crashed": any restart that races the marker-clearing write leaves the mark+    // behind on a perfectly healthy session, so a NORMAL restart could quarantine EVERYTHING (log:+    // "Recovering 6 session(s) from disk... Recovery complete. 0 session(s) active"). Crash+    // protection must not be able to delete healthy sessions — that is a worse failure than the+    // respawn loop it guards against.+    // The evidence we actually need is WHICH session was in flight when the process died. That is+    // one global sentinel, written before the risky ops and removed after, naming exactly one+    // session. A crash leaves it pointing at the poison file; everything else recovers untouched.+    if (_poisonSessionId === sessionId) {+      console.error(`Session "${sessionId}" was mid-recovery when pup died last boot (native crash) — QUARANTINING just this one; all other sessions still recover`);       deleteSessionFile(sessionId);       continue;     }-    try {-      fs.writeFileSync(sessionFilePath(sessionId), JSON.stringify({ ...sf, recoverAttempts: 1 }, null, 2));-    } catch (e) { /* if we can't mark it, still attempt — but the breaker only protects when the mark persists */ }+    try { fs.writeFileSync(RECOVERY_SENTINEL, sessionId); } catch (e) {}      // Check if Chrome is still alive     if (!isProcessAlive(pid)) {@@ -1181,6 +1201,10 @@ async function recoverSessions() {       deleteSessionFile(sessionId);     }   }+  // Every session got through its risky section, so nothing is in flight — drop the sentinel.+  // (Each iteration rewrites it, so a crash always leaves it naming the session that died.)+  try { if (fs.existsSync(RECOVERY_SENTINEL)) fs.unlinkSync(RECOVERY_SENTINEL); } catch (e) {}+  _poisonSessionId = null;   console.log(`Recovery complete. ${sessions.size} session(s) active.`); } @@ -7297,8 +7321,18 @@ server.listen(PORT, BIND_HOST, async () => {     console.error('Session recovery failed:', e.message);   } -  // v1.9.44: sweep orphaned AUMID keys AFTER recovery, so live sessions are known and kept.-  try { pruneStaleAumids(); } catch (e) {}+  // v1.9.45: sweep orphaned AUMID keys, but LATE. Running it straight after recoverSessions() looked+  // right and was wrong: when recovery legitimately ends with zero sessions (or is still settling),+  // "not a live session" matches EVERY key and the sweep deletes the whole set, unbranding windows+  // that were about to be re-stamped (observed: "pruned 12 stale keys" then keys: 0). Wait for the+  // session map to settle, and never sweep from an empty map — with no live sessions there is+  // nothing to distinguish stale from pending, so doing nothing is the correct, safe answer.+  setTimeout(() => {+    try {+      if (!sessions.size) { console.log('[identity] AUMID prune skipped — no live sessions to compare against'); return; }+      pruneStaleAumids();+    } catch (e) {}+  }, 60000);    // Periodic cleanup: every 5 minutes, remove stale session files with dead PIDs   setInterval(async () => {