Pup - the AI's own browser

@@ -358,13 +358,17 @@ const CAPTURE_LOGIN_JS = `(() => {       window.__pupCapturedLogin = rec;   // fallback for SPA logins that never navigate (sweep harvests it)     } catch (e) {}   }+  function formHasPwd(f){ try { return !!f && [].slice.call(f.querySelectorAll('input')).some(isPwd); } catch (e) { return false; } }   addEventListener('submit', function(e){ grab(e.target && e.target.tagName === 'FORM' ? e.target : null); }, true);   addEventListener('click', function(e){     try {-      var t = e.target && e.target.closest && e.target.closest('button, input[type=submit], [role=button]');+      var t = e.target && e.target.closest && e.target.closest('button, input[type=submit], [role=button], a');       if (!t) return;       var f = t.form || (t.closest && t.closest('form'));-      if (f && f.querySelector('input[type=password]')) grab(f);+      // v2.0.128 (consumer bug report #4): was a literal input[type=password] check, so a masked+      // type=text password field (Rails/Keycloak) never matched and capture missed on click. Use the+      // isPwd matcher; if there is no enclosing form, sweep the document (SPA login with no <form>).+      if (formHasPwd(f)) grab(f); else if (!f) grab(null);     } catch (e2) {}   }, true);   // v2.0.121 (John, Gusto miss): ALSO capture the instant the password field loses focus or@@ -373,6 +377,18 @@ const CAPTURE_LOGIN_JS = `(() => {   // and lost; grabbing on blur/change delivers the creds through the synchronous binding first.   addEventListener('change', function(e){ try { if (isPwd(e.target) && e.target.value) grab(e.target.form || null); } catch (e2) {} }, true);   addEventListener('blur', function(e){ try { if (isPwd(e.target) && e.target.value) grab(e.target.form || null); } catch (e2) {} }, true);+  // v2.0.128 (consumer bug report #4): a STANDARD Rails login often submits PROGRAMMATICALLY —+  // form.submit() (and requestSubmit's older cousins) does NOT dispatch the 'submit' event, so+  // submit/click capture silently missed it. Hook the prototype so a programmatic submit still grabs.+  try {+    var _origSubmit = HTMLFormElement.prototype.submit;+    HTMLFormElement.prototype.submit = function(){ try { if (formHasPwd(this)) grab(this); } catch (e) {} return _origSubmit.apply(this, arguments); };+  } catch (e) {}+  // Last-resort net: on Enter inside a login form, and on pagehide before ANY navigation tears the+  // page down, grab whatever password field is currently filled. Cheap, synchronous, dedup is+  // downstream (same host+user+pwd within the session is coalesced).+  addEventListener('keydown', function(e){ try { if (e.key === 'Enter' && isPwd(document.activeElement) && document.activeElement.value) grab(document.activeElement.form || null); } catch (e2) {} }, true);+  addEventListener('pagehide', function(){ try { grab(null); } catch (e2) {} }, true); })()`; let pupshotUrl = null; // v1.9.4: profileName -> in-flight launch Promise. A fresh profile isn't in `browsers` until its@@ -9282,13 +9298,25 @@ function resolveSessionOrError(args) {     };   } else if (explicit) {     // Caller asked for a specific sessionId we have NO record of.+    // v2.0.128 (consumer bug report #5): be HONEST about a bridge restart instead of leaving the+    // caller to guess whether their window was ever opened. If the bridge came up recently, say so+    // outright — "the bridge restarted Ns ago, your window did not survive" — and note that named+    // sessions auto-restore ONLY when their on-disk session file survived (pup relaunches them to+    // their last URL on the next verb); if this id isn't recovering, that file is gone.+    const _upS = Math.round(process.uptime());+    const _recentRestart = _upS < 180;     payload = {-      error: `Session "${requested}" not found in bridge. Either it was never opened, it was closed via pup_close_window, or the bridge restarted and the on-disk session file was cleaned.`,+      error: _recentRestart+        ? `The bridge restarted ${_upS}s ago and session "${requested}" did not survive — its on-disk session file was not found, so pup cannot auto-restore it. (Named sessions auto-restore across a restart only while their session file persists; this one's is gone, or the window was opened by a build that didn't persist it.)`+        : `Session "${requested}" not found in bridge. Either it was never opened, it was closed via pup_close_window, or the bridge restarted and the on-disk session file was cleaned.`,       errorCode: 'session_not_found',+      bridgeUptimeSec: _upS,+      bridgeRecentlyRestarted: _recentRestart,       currentSessions,       liveSessions,       disconnectedSessions,       _hint:+        (_recentRestart ? `The window is gone — reopen it: pup_open_window {"sessionId":"${requested}","url":"..."} (pup re-keys it to this id and remembers it). ` : '') +         `Live sessions right now: [${liveSessions.join(', ') || '(none)'}]. ` +         (disconnectedSessions.length > 0           ? `Disconnected sessions awaiting rescan: [${disconnectedSessions.join(', ')}] — try \`pup_rescan\` first. `@@ -11537,9 +11565,10 @@ const server = http.createServer(async (req, res) => {             V('pup_back', 'History back.', { sessionId: 'required str' }, 20, { sessionId: 'work' }),             V('pup_forward', 'History forward.', { sessionId: 'required str' }, 20, { sessionId: 'work' }),             V('pup_reload', 'Reload the active tab.', { sessionId: 'required str' }, 30, { sessionId: 'work' }),-            V('pup_click', 'Click an element (by selector or coords).', { sessionId: 'required str', selector: 'optional str', x: 'optional int', y: 'optional int' }, 20, { sessionId: 'work', selector: 'button#go' }),+            V('pup_click', 'Click an element (by selector or coords). Selector clicks LOCATE the element once then click by its center coordinates (fast on huge/animated DOMs, where page.click used to hit the 15s wall). Response includes method. If a React toggle/like/heart does NOT change state after a normal click, that control ignores trusted clicks — retry with domClick:true to fire a synthetic element.click().', { sessionId: 'required str', selector: 'optional str', x: 'optional int', y: 'optional int', domClick: 'optional bool — dispatch a synthetic element.click() instead of a trusted CDP click; for React controls that ignore trusted clicks', clickCount: 'optional int 1-3', button: 'optional "left"|"right"|"middle"' }, 20, { sessionId: 'work', selector: 'button#go' }),             V('pup_assisted_click', 'RESPECTFUL real-cursor click for a surface pup CDP cannot drive (a NATIVE OS dialog, or a cross-origin FedCM "Sign in with Google" account chooser that rejects every in-page click). Do NOT hand-roll raise+desktop_click for these — use THIS. It: captions the reason + toasts BEFORE touching the screen; foregrounds; then WAITS, detecting if the user alt-tabbed away and holding (captioning "Adom is waiting — click back when ready") until the user is actually on the window; then runs the TIGHT real-cursor hover->click that FedCM accepts (interleaving any screenshot/eval between raise and click makes the click silently no-op); captions each step. Times out as user_not_ready clicking NOTHING if they never return. First take a desktop_screenshot_window of the session to read the target\'s WINDOW-pixel x,y.', { sessionId: 'required str', x: 'required int — WINDOW-image pixel of the target (from desktop_screenshot_window)', y: 'required int', reason: 'required str — WHY you must click (shown to the user as a caption)', label: 'required str — WHAT you are clicking (e.g. "your Google account")', maxWaitMs: 'optional int — how long to wait for the user (default 120000)' }, 130, { sessionId: 'adsk', x: 900, y: 379, reason: 'clicking your Google account to finish signing into Autodesk', label: 'your Google account' }),-            V('pup_type', 'Type text into a field. Pass clear:true to REPLACE the existing value instead of appending (this is what you want on any prefilled/React form — it selects-all then deletes with real key events, which React-controlled inputs honour, unlike setting .value).', { sessionId: 'required str', text: 'required str', selector: 'optional str', clear: 'optional bool — empty the field first (replace, not append)' }, 20, { sessionId: 'work', selector: '#user', text: 'adominc1', clear: true }),+            V('pup_type', 'Type text into a field via real CDP keystrokes (isTrusted). Reads the value back and returns valueLanded — real keystrokes only register in a FOREGROUNDED window, so if valueLanded:false the field stayed empty (the silent no-op): use pup_fill (background-safe) for a non-React input, or pup_login {realKeystrokes:true} for a login form. Pass clear:true to REPLACE instead of appending.', { sessionId: 'required str', text: 'required str', selector: 'optional str', clear: 'optional bool — empty the field first (replace, not append)' }, 20, { sessionId: 'work', selector: '#user', text: 'adominc1', clear: true }),+            V('pup_fill', 'Set a field\'s value in the BACKGROUND (no foreground, no keystrokes): writes .value via the native setter and fires input+change so React/Vue/Angular controlled inputs pick it up. The fix for pup_type\'s silent no-op on a backgrounded window. Reads the value back and returns valueLanded. For React-MASKED password fields that reject programmatic writes, use pup_login {realKeystrokes:true} instead.', { sessionId: 'required str', selector: 'required str', text: 'required str' }, 20, { sessionId: 'work', selector: '#email', text: '[email protected]' }),             V('pup_press_key', 'Press a key, optionally with modifier chords held (Ctrl+A, Ctrl+V, Shift+Tab).', { sessionId: 'required str', key: 'required str', modifiers: 'optional array — any of ["Control","Shift","Alt","Meta"] (aliases ctrl/cmd/command/option accepted); held down around the key press' }, 15, { sessionId: 'work', key: 'a', modifiers: ['Control'] }),             V('pup_NATIVE_INPUT_WARNING', '⛔ READ BEFORE USING AD\'s desktop_click / desktop_type / desktop_press_key ON A PUP WINDOW. Native OS input does NOT target a window — it goes to WHATEVER WINDOW HAS OS FOCUS at that instant. This user runs many AI threads and alt-tabs away from a foregrounded window almost immediately. If you fire native input without warning them, your keystrokes land in THEIR document (this happened: a username was typed into the user\'s own window). REQUIRED: notify_user toast + desktop_caption FIRST, tell them explicitly "don\'t type or click until I say done", wait until the window is genuinely foreground, re-verify immediately before EACH event, and caption when done. pup_assisted_click already does all of this — use it. And ALWAYS prefer CDP input (pup_click/pup_type/pup_hover): it needs no foreground and disturbs nobody. A native verb returning ok:true means the event was dispatched to the OS, NOT that it reached your window.', {}, 0, {}),             V('pup_set_viewport', 'Pin the page viewport (CDP device-metrics) to an exact size for deterministic screenshots — OR clear that pin. ⚠ A pin PERSISTS: it survives navigation and hard reload, and while set the window stops tracking OS resizing (a human resizing it sees no reflow, which looks exactly like an app layout bug). ALWAYS clear it when done. Reported as emulatedViewport in pup_status / pup_list_windows.', { sessionId: 'required str', width: 'int — required unless reset', height: 'int — required unless reset', deviceScaleFactor: 'optional number (default 1)', reset: 'optional bool — CLEAR the override and track the OS window again (width/height not needed)' }, 15, { sessionId: 'work', reset: true }),@@ -13892,18 +13921,54 @@ const server = http.createServer(async (req, res) => {             const _intent = (typeof args.intent === 'string' && args.intent.trim())               ? args.intent.trim()               : (args.reason && String(args.reason).trim() ? String(args.reason).trim() : (_tgt ? `Clicking ${_tgt}` : 'Clicking'));-            if (args.selector) {+            let _method = 'cdp';+            if (args.selector && args.domClick === true) {+              // v2.0.128 (consumer bug report #2): some React controls (heart/like/vote toggles)+              // ignore a CDP "trusted" mouse click but DO fire on a synthetic element.click(). This+              // is the opposite of pup's usual advice, so it's opt-in — the miss-hint below points+              // callers here when a trusted click produced no state change.+              _method = 'domClick';+              const dr = await withTimeout(tab.page.evaluate((sel) => {+                const el = document.querySelector(sel);+                if (!el) return { found: false };+                el.scrollIntoView({ block: 'center', inline: 'center' });+                el.click();+                return { found: true };+              }, args.selector), 8_000, 'domClick.evaluate');+              if (!dr.found) { sendJSON(res, { success: false, errorCode: 'selector_not_found', error: `pup_click(domClick): selector "${args.selector}" not found.`, _hint: `Probe: pup_eval {"expr":"!!document.querySelector('${args.selector}')"}` }); return; }+            } else if (args.selector) {+              // v2.0.128 (consumer bug report #3): page.click(selector) re-resolves the element and+              // waits on Puppeteer's visibility machinery, which crawls on huge/animated DOMs and hit+              // the 15s wall — while a plain coordinate click on the SAME element returned instantly.+              // So: locate the element ONCE (scroll into view, read its center), then click by those+              // coordinates — the fast path the caller proved works. Only fall back to page.click when+              // the box can't be resolved (0-size / not laid out yet).               let _box = null;-              try { _box = await tab.page.$eval(args.selector, el => { const r = el.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 }; }); } catch (e) {}-              if (_cursor && _box) { await showPupCursor(tab.page, _box.x, _box.y, { label: _intent }); await new Promise(r => setTimeout(r, 440)); }-              // v1.9.197 BUG 2: clickCount was accepted and silently ignored (a triple-click did a-              // single click and selected nothing). Honour it — 3 selects the field's text.-              await withTimeout(tab.page.click(args.selector, _clickOpts), 10_000, 'page.click');+              try {+                _box = await withTimeout(tab.page.evaluate((sel) => {+                  const el = document.querySelector(sel);+                  if (!el) return null;+                  el.scrollIntoView({ block: 'center', inline: 'center' });+                  const r = el.getBoundingClientRect();+                  if (r.width < 1 || r.height < 1) return null;+                  return { x: r.x + r.width / 2, y: r.y + r.height / 2 };+                }, args.selector), 6_000, 'locate.box');+              } catch (e) {}+              if (_box) {+                if (_cursor) { await showPupCursor(tab.page, _box.x, _box.y, { label: _intent }); await new Promise(r => setTimeout(r, 440)); }+                _method = 'coordinate';+                await withTimeout(tab.page.mouse.click(_box.x, _box.y, _clickOpts), 10_000, 'mouse.click(box)');+              } else {+                // element not found/visible yet — bounded wait, then Puppeteer's own click (honours clickCount).+                _method = 'page.click';+                await withTimeout(tab.page.click(args.selector, _clickOpts), 12_000, 'page.click');+              }             } else {               if (_cursor) { await showPupCursor(tab.page, Number(args.x), Number(args.y), { label: _intent }); await new Promise(r => setTimeout(r, 440)); }+              _method = 'coordinate';               await withTimeout(tab.page.mouse.click(Number(args.x), Number(args.y), _clickOpts), 10_000, 'mouse.click');             }-            sendJSON(res, { success: true, output: JSON.stringify({ ok: true, clicked: args.selector || { x: Number(args.x), y: Number(args.y) }, sessionId: resolvedId, tabId, _hint: 'Click dispatched (isTrusted, CDP-level). Verify the effect with pup_eval (e.g. document.title or a state probe) — a click that hit nothing still returns ok.' }) });+            sendJSON(res, { success: true, output: JSON.stringify({ ok: true, clicked: args.selector || { x: Number(args.x), y: Number(args.y) }, method: _method, sessionId: resolvedId, tabId, _hint: (args.selector && _method !== 'domClick') ? 'Click dispatched (isTrusted). Verify the effect with pup_eval (a state probe) — a click that hit nothing still returns ok. If a React toggle/like/heart did NOT change state, that control ignores trusted clicks: retry pup_click {selector, domClick:true} to fire the synthetic handler.' : 'Click dispatched. Verify the effect with pup_eval (e.g. document.title or a state probe) — a click that hit nothing still returns ok.' }) });           } catch (e) {             sendJSON(res, { success: false, errorCode: 'click_failed', error: `pup_click failed: ${e.message}`, _hint: args.selector ? `Selector "${args.selector}" may not exist/be visible. Probe it first: pup_eval {"expr":"!!document.querySelector('${args.selector}')"} — or click by x/y from a pup_screenshot.` : 'x/y click failed — coordinates are page-viewport pixels; take a pup_screenshot to aim.' });           }@@ -13978,12 +14043,85 @@ const server = http.createServer(async (req, res) => {             }             if (args.selector) await withTimeout(tab.page.type(args.selector, args.text), 15_000, 'page.type');             else await withTimeout(tab.page.keyboard.type(args.text), 15_000, 'keyboard.type');-            sendJSON(res, { success: true, output: JSON.stringify({ ok: true, typed: args.text.length + ' chars', into: args.selector || 'focused element', sessionId: resolvedId, tabId, _hint: 'Typed via CDP (isTrusted). Verify with pup_eval reading the field value.' }) });+            // v2.0.128 (consumer bug report #1 — highest priority): pup_type reported ok:true on a+            // field that stayed EMPTY, so a sign-in silently no-opped while pup claimed success. Read+            // the value back when we have a selector, and if what we typed did not land, still return+            // ok:true (the keystrokes WERE dispatched) but flag valueLanded:false with the real reason:+            // CDP/page.keyboard keystrokes only register in a FOREGROUNDED window, and this window is+            // very likely in the background. Point at the two working paths.+            let _valueLanded = null, _readLen = null;+            if (args.selector) {+              try {+                _readLen = await withTimeout(tab.page.$eval(args.selector, el => {+                  if (el == null) return -1;+                  if (typeof el.value === 'string') return el.value.length;+                  if (el.isContentEditable) return (el.textContent || '').length;+                  return -2; // element exists but has no value/textContent we can read+                }), 5_000, 'verify.$eval');+                if (_readLen >= 0) _valueLanded = _readLen >= Math.min(args.text.length, 1);+              } catch (e) { _readLen = null; }+            }+            const _emptyMiss = _valueLanded === false && args.text.length > 0;+            const _out = { ok: true, typed: args.text.length + ' chars', into: args.selector || 'focused element', sessionId: resolvedId, tabId };+            if (args.selector) { _out.valueLanded = _valueLanded; _out.fieldValueLength = _readLen; }+            _out._hint = _emptyMiss+              ? '⚠ KEYSTROKES DISPATCHED BUT THE FIELD IS STILL EMPTY — this is the silent no-op. CDP/page.keyboard keystrokes ONLY register when the window is FOREGROUNDED, and this window is almost certainly in the background. Two fixes: (1) for a NON-React input, set the value directly — pup_eval will not (expression-only), so use pup_fill {selector,text} which sets .value via the native setter + dispatches input/change and works in the background; (2) for a login form (React-masked fields, real keystrokes required) use pup_login {username,password,realKeystrokes:true}, which foregrounds the window automatically. Do NOT trust a bare ok:true from pup_type on a login field — check valueLanded.'+              : (_valueLanded === true ? 'Typed via CDP (isTrusted); value verified present in the field.' : 'Typed via CDP (isTrusted). Could not read the field back to verify (no selector, or non-standard element) — confirm with a screenshot before assuming it landed.');+            sendJSON(res, { success: true, output: JSON.stringify(_out) });           } catch (e) {             sendJSON(res, { success: false, errorCode: 'type_failed', error: `pup_type failed: ${e.message}`, _hint: args.selector ? `Selector "${args.selector}" may not exist. Probe: pup_eval {"expr":"!!document.querySelector('${args.selector}')"}` : 'No selector given — ensure an element has focus (pup_click it first).' });           }           return;         }+        // v2.0.128 (consumer bug report #1): the BACKGROUND-safe fallback that pup_type's miss-hint+        // points at. Sets the field's value via the native property setter and dispatches input ++        // change so React/Vue/Angular controlled inputs pick it up — no foreground, no keystrokes.+        // Works for the common "logged-out form filled while the window is in the background" case.+        // For React-MASKED password fields that ignore programmatic value writes, pup_login+        // {realKeystrokes:true} is still the answer (it foregrounds + sends real keys). pup_fill+        // ALWAYS reads the value back and reports valueLanded so a caller never gets a false ok.+        case 'pup_fill': {+          if (!args.selector || typeof args.text !== 'string') {+            sendJSON(res, { success: false, error: 'pup_fill needs "selector" and "text" (string). It sets .value via the native setter + fires input/change (React-safe, works in the background). Usage: { "sessionId": "...", "selector": "#email", "text": "[email protected]" }' });+            return;+          }+          let session, resolvedId;+          try { ({ session, resolvedId } = await recoverOrRelaunchSession(args)); }+          catch (e) { sendSessionResolveError(res, e); return; }+          let tab, tabId;+          try { ({ tab, tabId } = resolveTabOrError(session, resolvedId, args)); }+          catch (e) { sendTabResolveError(res, e); return; }+          try {+            const r = await withTimeout(tab.page.evaluate((sel, val) => {+              const el = document.querySelector(sel);+              if (!el) return { found: false };+              // Use the prototype's native value setter so React's descriptor override doesn't swallow it.+              try {+                const proto = el instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;+                const desc = Object.getOwnPropertyDescriptor(proto, 'value');+                if (desc && desc.set) desc.set.call(el, val); else el.value = val;+              } catch (e) { try { el.value = val; } catch (e2) {} }+              el.dispatchEvent(new Event('input', { bubbles: true }));+              el.dispatchEvent(new Event('change', { bubbles: true }));+              const got = (typeof el.value === 'string') ? el.value : (el.textContent || '');+              return { found: true, landedLen: got.length, matched: got === val };+            }, args.selector, args.text), 8_000, 'pup_fill.evaluate');+            if (!r.found) {+              sendJSON(res, { success: false, errorCode: 'selector_not_found', error: `pup_fill: selector "${args.selector}" not found.`, _hint: `Probe it: pup_eval {"expr":"!!document.querySelector('${args.selector}')"}` });+              return;+            }+            const _landed = r.matched === true || r.landedLen >= Math.min(args.text.length, 1);+            sendJSON(res, { success: true, output: JSON.stringify({+              ok: true, filled: args.selector, valueLanded: _landed, fieldValueLength: r.landedLen, sessionId: resolvedId, tabId,+              _hint: _landed+                ? 'Set via native setter + input/change (background-safe). If a React SUBMIT still rejects it as empty, the field is masked/guarded — use pup_login {username,password,realKeystrokes:true} which foregrounds + sends real keystrokes.'+                : '⚠ Value did not stick — this field guards against programmatic writes (React-masked password fields do this). Use pup_login {realKeystrokes:true} (foregrounds + real keys) instead.'+            }) });+          } catch (e) {+            sendJSON(res, { success: false, errorCode: 'fill_failed', error: `pup_fill failed: ${e.message}`, _hint: 'The page may have navigated. Re-check the field with pup_eval, or retry.' });+          }+          return;+        }         case 'pup_press_key': {           if (!args.key) {             sendJSON(res, { success: false, error: 'pup_press_key needs "key" — a puppeteer KeyInput name like "Enter", "Tab", "Escape", "ArrowDown", "a". Usage: { "sessionId": "...", "key": "Enter" }' });