Pup - the AI's own browser

@@ -1486,6 +1486,70 @@ function osSendWindowToBack(profileName) {   } catch (e) { console.error(`[bg] send-to-back failed for "${profileName}": ${e.message}`); return false; } } +// v1.8.23: REAL taskbar flash (FlashWindowEx) for a session's window — the orange+// nudge that tells the user "the agent touched this window" WITHOUT stealing focus.+// FLASHW_TRAY|FLASHW_TIMERNOFG flashes until the window is focused, then auto-clears+// (Windows does the clear-on-focus for us). Skips if the window is already foreground.+// Fire-and-forget (async exec) so it never adds latency to the driving verb.+function osFlashWindow(profileName) {+  if (process.platform !== 'win32') return;+  const be = browsers.get(profileName);+  const pid = (be && be.browser && (be.browser._detachedPid || (be.browser.process() && be.browser.process().pid))) || null;+  if (!pid) return;+  try {+    const { exec } = require('child_process');+    const psScript = path.join(require('os').tmpdir(), `_adom_flash_${pid}.ps1`);+    const ps = [+      'Add-Type @"',+      'using System; using System.Runtime.InteropServices;',+      'public class WinFlash {',+      '  [StructLayout(LayoutKind.Sequential)] public struct FLASHWINFO { public uint cbSize; public IntPtr hwnd; public uint dwFlags; public uint uCount; public uint dwTimeout; }',+      '  [DllImport("user32.dll")] public static extern bool FlashWindowEx(ref FLASHWINFO pwfi);',+      '  [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();',+      '  public static void Flash(IntPtr h){ if(h==IntPtr.Zero) return; if(GetForegroundWindow()==h) return;',+      '    FLASHWINFO fi=new FLASHWINFO(); fi.cbSize=(uint)Marshal.SizeOf(fi); fi.hwnd=h; fi.dwFlags=0x2|0xC; fi.uCount=0; fi.dwTimeout=0; FlashWindowEx(ref fi); }',+      '}',+      '"@',+      `$targetPid = ${pid}`,+      '$p = Get-Process -Id $targetPid -ErrorAction SilentlyContinue',+      'if ($p -and $p.MainWindowHandle -ne 0) { [WinFlash]::Flash($p.MainWindowHandle) }',+      'else { Get-CimInstance Win32_Process -Filter "ParentProcessId=$targetPid" -ErrorAction SilentlyContinue | Where-Object { $_.Name -in @("chrome.exe","msedge.exe") } | ForEach-Object { $cp = Get-Process -Id $_.ProcessId -ErrorAction SilentlyContinue; if ($cp -and $cp.MainWindowHandle -ne 0) { [WinFlash]::Flash($cp.MainWindowHandle) } } }',+    ].join('\r\n');+    fs.writeFileSync(psScript, ps);+    exec(`powershell.exe -NoProfile -ExecutionPolicy Bypass -File "${psScript}"`, { timeout: 6000 }, () => {});+  } catch (e) {}+}++// v1.8.23: verbs that MUTATE a session's visible state → auto-flash the window so the+// user knows the agent changed it. evaluate counts UNLESS {readOnly:true}.+const MUTATING_VERBS = new Set(['browser_open_window', 'browser_open_tab', 'browser_navigate', 'browser_reload', 'browser_back', 'browser_forward', 'browser_switch_tab', 'browser_click', 'browser_type', 'browser_press_key', 'browser_input_dispatch', 'browser_eval', 'browser_evaluate']);++// Debounced auto-flash: one flash per window per ~5s burst of verbs, re-armed after+// quiet. {silent:true} on the verb OR session._flashSilent (session default) suppresses.+const _flashState = new Map(); // profileName -> last flash ts+function scheduleAutoFlash(session, opts) {+  opts = opts || {};+  if (process.platform !== 'win32' || !session) return;+  if (opts.silent || session._flashSilent) return;+  const key = session.profileName;+  const now = Date.now();+  if (now - (_flashState.get(key) || 0) < 5000) return; // coalesce the burst+  _flashState.set(key, now);+  osFlashWindow(key);+}++// Record what the agent just did to a session, for lastAgentUpdate (browser_list_tabs /+// list_windows / status) and the "● " title marker — so the user + Session Monitor can+// see WHICH window/tab, WHAT verb, and the optional human note, WHEN.+function recordAgentUpdate(session, verb, args, tabId) {+  if (!session) return;+  const note = (args && typeof args.updateNote === 'string' && args.updateNote.trim()) ||+               (args && typeof args.note === 'string' && args.note.trim()) || null;+  const upd = { tabId: tabId || session.activeTabId || null, verb, note, ts: Date.now() };+  session._lastAgentUpdate = upd;+  if (upd.tabId) { const t = (session.tabs || []).find(x => x.tabId === upd.tabId); if (t) t.lastAgentUpdate = upd; }+}+ // Resize/position a session's window via CDP (works for detached+connected). Setting // explicit bounds also un-maximizes the window (--start-maximized is otherwise the // default). x/y/width/height in screen pixels; any subset is honored.@@ -2537,6 +2601,9 @@ async function getSessionInfo(sessionId, session) {     // v1.8.16: advisory ownership — which AI thread/task opened this window.     owner: session.owner || null,     ageMinutes: session.createdAt ? Math.round((Date.now() - session.createdAt) / 60000) : null,+    // v1.8.23: last thing the agent did here {tabId, verb, note, ts} — for the user ++    // Session Monitor to see WHICH window, WHAT, and WHEN.+    lastAgentUpdate: session._lastAgentUpdate || null,   };   // One-shot: surface launch-time metadata (restored-tab cleanup count)   // so callers can verify session-restore prevention worked. Drop after read.@@ -2610,6 +2677,20 @@ const server = http.createServer(async (req, res) => {       const args = body.args || {};       res._adomCommand = command; // v1.0.16: lets sendJSON inject this verb's _hint/related +      // v1.8.23: AUTO-FLASH + agent-update tracking for state-mutating verbs on an EXISTING+      // session (browser_open_window flashes from inside its own handler, once the session+      // exists). evaluate is read-only iff {readOnly:true}. {silent:true} per verb or the+      // session default suppresses. Runs BEFORE the handler — signals "the agent is+      // touching this window" regardless of the verb's outcome.+      if (MUTATING_VERBS.has(command) && command !== 'browser_open_window' && command !== 'browser_open_tab') {+        const isReadOnlyEval = (command === 'browser_eval' || command === 'browser_evaluate') && args.readOnly;+        if (!isReadOnlyEval) {+          const sid = args.sessionId || activeSessionId;+          const s = sid && sessions.get(sid);+          if (s) { recordAgentUpdate(s, command, args, args.tabId || s.activeTabId); scheduleAutoFlash(s, { silent: !!args.silent }); }+        }+      }+       switch (command) {         // v1.0.15: self-describe — AD's "Verbs" tab POSTs {command:'browser_describe'}         // and renders this verbatim: one entry per documented verb with@@ -2625,7 +2706,7 @@ const server = http.createServer(async (req, res) => {                      hint: m.hint || summary, related: m.related || [], pitfalls: m.pitfalls || [] };           };           sendJSON(res, { success: true, bridge: 'puppeteer', verbs: [-            V('browser_open_window', 'Open a generic pup window — a fresh isolated Chrome/Edge process (never your real signed-in browser), fully drivable via CDP. ALWAYS opens in the BACKGROUND (does not steal focus; never disturbs the user\'s main work; still fully screenshottable/drivable) UNLESS you pass foreground:true — do that ONLY when the user explicitly wants to watch. Pass width+height (+ optional x,y, screen px) to place it at a specific size/position for screenshot framing — this NO LONGER foregrounds it (sizing stays in the background too; default unsized is maximized). profile defaults to sessionId. OWNERSHIP: sessionIds are shared across ALL AI threads — use a task-prefixed sessionId + owner:"<your-task>" so pup can refuse other threads stealing your window (reusing an existing id with a new url navigates it away; declared cross-owner reuse errors with session_owned_by_another_thread unless takeover:true). (For the user\'s REAL Chrome/Edge, use the Adom extension\'s nbrowser_* verbs.)', { sessionId: 'required str — task-prefix it (e.g. "mytask-web"); generic ids collide across threads', url: 'optional str', profile: 'optional str — defaults to sessionId', owner: 'optional str — your thread/task label; lets pup protect this window from other threads', takeover: 'optional bool — claim a window owned by another thread (only when the user explicitly asks)', foreground: 'optional bool — the ONLY way to show the window (default false = background, always, even when sized)', width: 'optional int px — sizes for screenshot framing, stays background', height: 'optional int px', x: 'optional int px', y: 'optional int px' }, 60, { sessionId: 'mytask-web', owner: 'mytask', url: 'https://example.com', width: 960, height: 540 }),+            V('browser_open_window', 'Open a generic pup window — a fresh isolated Chrome/Edge process (never your real signed-in browser), fully drivable via CDP. ALWAYS opens in the BACKGROUND (does not steal focus; never disturbs the user\'s main work; still fully screenshottable/drivable) UNLESS you pass foreground:true — do that ONLY when the user explicitly wants to watch. Pass width+height (+ optional x,y, screen px) to place it at a specific size/position for screenshot framing — this NO LONGER foregrounds it (sizing stays in the background too; default unsized is maximized). profile defaults to sessionId. OWNERSHIP: sessionIds are shared across ALL AI threads — use a task-prefixed sessionId + owner:"<your-task>" so pup can refuse other threads stealing your window (reusing an existing id with a new url navigates it away; declared cross-owner reuse errors with session_owned_by_another_thread unless takeover:true). (For the user\'s REAL Chrome/Edge, use the Adom extension\'s nbrowser_* verbs.)', { sessionId: 'required str — task-prefix it (e.g. "mytask-web"); generic ids collide across threads', url: 'optional str', profile: 'optional str — defaults to sessionId', owner: 'optional str — your thread/task label; lets pup protect this window from other threads', takeover: 'optional bool — claim a window owned by another thread (only when the user explicitly asks)', foreground: 'optional bool — the ONLY way to show the window (default false = background, always, even when sized)', width: 'optional int px — sizes for screenshot framing, stays background', height: 'optional int px', x: 'optional int px', y: 'optional int px', updateNote: 'optional str — a short "what I did" note, surfaced back in list_tabs/list_windows as lastAgentUpdate so the user knows what changed', autoFlash: 'optional bool (default true) — mutating verbs auto-flash this window\'s taskbar (debounced, never steals focus) so the user notices the agent touched it; pass false to make this window silent by default', silent: 'optional bool — suppress the auto-flash for just this call' }, 60, { sessionId: 'mytask-web', owner: 'mytask', url: 'https://example.com', width: 960, height: 540 }),             V('browser_open_tab', 'Open a new tab in a session.', { sessionId: 'required str', url: 'optional str' }, 30, { sessionId: 'work', url: 'https://example.com' }),             V('browser_close_tab', 'Close a tab.', { sessionId: 'required str', tabId: 'optional str (default active)' }, 15, { sessionId: 'work' }),             V('browser_close_window', 'Close a window/session (its tabs go too).', { sessionId: 'required str' }, 20, { sessionId: 'work' }),@@ -2654,8 +2735,9 @@ const server = http.createServer(async (req, res) => {             V('browser_prewarm', 'Install Chrome for Testing WITHOUT opening a window, and pin it as the default. Use when you specifically want CfT rather than the machine\'s installed Chrome/Edge.', { wait: 'optional bool — block until installed', setDefault: 'optional bool (default true) — pin CfT as the default' }, 240, {}),             V('browser_use', 'Pin (or clear) the browser pup drives by default: chrome | edge | cft | auto (or an explicit executablePath). "cft" fetches Chrome for Testing then pins it. With {browser:"chrome", install:true} it DOWNLOADS & INSTALLS real Google Chrome on an Edge-only box, then pins it — SILENT on a normal PC; on a locked-down box (VM/managed device) it needs a Windows approval, so the bridge pops a native "click YES on the UAC" notification and, if it expires, re-notifies with a one-tap "Approve now" (all autonomous — no AI turn). Poll browser_readiness.chromeStablePhase: installing → awaiting_uac → ready. Persists across restarts; launch still spawn-verifies + falls back.', { browser: '"chrome" | "edge" | "cft" | "auto"', install: 'optional bool — with browser:"chrome", install real Chrome if missing', elevate: 'optional "auto"(default)|true|false — auto: quiet install, prompt only if it needs admin; false: never prompt; true: go straight to the prompt', executablePath: 'optional — pin an exact browser binary' }, 240, {}),             V('browser_lower_os_window', 'Send a pup window to the BACKGROUND (behind the user\'s windows) without stealing focus — still fully drivable. The default for open; use this to re-hide a window you foregrounded.', { sessionId: 'optional str' }, 15, {}),-            V('browser_raise_os_window', 'Bring a pup window to the FOREGROUND (user\'s screen). Use SPARINGLY — only when the user must SEE it (a demo, recording, "show me"). Alias: browser_focus_window.', { sessionId: 'optional str' }, 15, {}),-            V('browser_alert_window', 'Flash the taskbar icon for a pup window — a non-intrusive "it\'s ready/done" nudge that does NOT steal focus. Prefer this over raising the window.', { sessionId: 'optional str' }, 15, {}),+            V('browser_raise_os_window', 'Bring a pup window to the FOREGROUND (user\'s screen). Use SPARINGLY — only when the user EXPLICITLY must SEE it (a demo, recording, "show me"). It STEALS focus and can leak a mid-typed keystroke into the page — to just signal "I updated this window", use browser_alert_window (or rely on the auto-flash), NOT raise. pup blurs the page\'s focused element before raising to reduce the keystroke leak. Alias: browser_focus_window.', { sessionId: 'optional str' }, 15, {}),+            V('browser_alert_window', 'Flash the taskbar icon for a pup window — a non-intrusive "it\'s ready/DONE" nudge that does NOT steal focus. NOTE (v1.8.23): mutating verbs already AUTO-FLASH (debounced, one per ~5s burst per window) so the user always knows the agent touched a window — use this for an explicit end-of-task "it\'s ready" nudge on top of that. STRONGLY prefer this (or the auto-flash) over browser_raise_os_window, which steals focus and can leak a mid-typed keystroke into the page.', { sessionId: 'optional str' }, 15, {}),+            V('browser_render_html', 'Render a local HTML string in the tab via page.setContent — UTF-8-correct (a data: URL mojibakes accented chars without an explicit charset). Great for previewing generated markup without a temp file. Auto-flashes the window.', { sessionId: 'required str', html: 'required HTML string', waitUntil: 'optional (domcontentloaded default)', updateNote: 'optional note for lastAgentUpdate', silent: 'optional bool — suppress the auto-flash' }, 30, { sessionId: 'work', html: '<h1>Café</h1>' }),             // Session/tab/window management + probes (were routed but undocumented).             V('browser_status', 'List all open pup sessions + their tabs (NOT a readiness check — use browser_readiness for that).', {}, 10, {}),             V('browser_list_windows', 'List every open session/window with its tabs.', {}, 10, {}),@@ -2796,6 +2878,15 @@ const server = http.createServer(async (req, res) => {             try { await session.page.setViewport({ width: Math.round(w), height: Math.round(h), deviceScaleFactor: Number(args.deviceScaleFactor) || 1 }); } catch (e) {}           }           if (!foreground) { try { osSendWindowToBack(args.profile); } catch (e) {} }+          // v1.8.23: session-level auto-flash default (autoFlash:false → this window never+          // auto-flashes, e.g. a chatty automation loop the user isn't watching).+          if (session && args.autoFlash === false) session._flashSilent = true;+          // Flash on open (a NEW window appearing IS the update) — unless foregrounded+          // (already on screen) or suppressed. Also stamp the first agent update.+          if (session) {+            recordAgentUpdate(session, 'browser_open_window', args, session.activeTabId);+            if (!foreground) scheduleAutoFlash(session, { silent: !!args.silent });+          }           // Verbose browser report: which browser we actually drove, what else is available.           const brep = browserPickReport();           const bgHint = foreground@@ -2967,13 +3058,40 @@ const server = http.createServer(async (req, res) => {             const tabId = await addTabToSession(sessionId, args.url);             const session = sessions.get(sessionId);             const tab = getTabById(session, tabId);+            recordAgentUpdate(session, 'browser_open_tab', args, tabId); // v1.8.23+            scheduleAutoFlash(session, { silent: !!args.silent });             const info = await describeTab(tab, session.activeTabId);-            sendJSON(res, { success: true, output: JSON.stringify({ ok: true, sessionId, ...info }) });+            // v1.8.23: return tabId + tabCount + activeTabId at the TOP level (was only inside `info`).+            sendJSON(res, { success: true, output: JSON.stringify({ ok: true, sessionId, tabId, tabCount: session.tabs.length, activeTabId: session.activeTabId, ...info }) });           } catch (e) {             sendJSON(res, { success: false, error: e.message });           }           return;         }+        case 'browser_render_html': {+          // v1.8.23: render a local HTML string in the tab via page.setContent — which+          // handles encoding correctly (UTF-8), unlike a data: URL that mojibakes accented+          // chars without an explicit charset. Pass {html}. Great for previewing generated+          // markup without writing a temp file or fighting data:-URL escaping.+          let session, resolvedId;+          try { ({ session, resolvedId } = await recoverOrRelaunchSession(args)); }+          catch (e) { sendSessionResolveError(res, e); return; }+          let tab, tabId;+          try { ({ tab, tabId } = resolveTabOrError(session, resolvedId, args)); }+          catch (e) { sendTabResolveError(res, e); return; }+          if (typeof args.html !== 'string') {+            sendJSON(res, { success: false, error: 'browser_render_html requires an html string.', _hint: 'e.g. {"sessionId":"...","html":"<h1>Café</h1>"} — setContent renders it as UTF-8 (a data: URL would mojibake the é).' });+            return;+          }+          try {+            await withTimeout(tab.page.setContent(args.html, { waitUntil: args.waitUntil || 'domcontentloaded', timeout: 20000 }), 25_000, 'page.setContent');+            recordAgentUpdate(session, 'browser_render_html', args, tabId);+            scheduleAutoFlash(session, { silent: !!args.silent });+            let title = null; try { title = await tab.page.title(); } catch {}+            sendJSON(res, { success: true, output: JSON.stringify({ ok: true, sessionId: resolvedId, tabId, title, bytes: args.html.length, _hint: 'Rendered your HTML in the tab (UTF-8, no data:-URL mojibake). Screenshot it or drive it like any page.' }) });+          } catch (e) { sendJSON(res, { success: false, error: `browser_render_html failed: ${e.message}` }); }+          return;+        }         case 'browser_switch_tab': {           const sessionId = args.sessionId || activeSessionId;           if (!sessionId || !sessions.has(sessionId)) {@@ -3031,9 +3149,11 @@ const server = http.createServer(async (req, res) => {           const session = sessions.get(sessionId);           const tabs = [];           for (const t of session.tabs) {-            tabs.push(await describeTab(t, session.activeTabId));+            const info = await describeTab(t, session.activeTabId);+            if (t.lastAgentUpdate) info.lastAgentUpdate = t.lastAgentUpdate; // v1.8.23: per-tab+            tabs.push(info);           }-          sendJSON(res, { success: true, output: JSON.stringify({ ok: true, sessionId, count: tabs.length, activeTabId: session.activeTabId, tabs }) });+          sendJSON(res, { success: true, output: JSON.stringify({ ok: true, sessionId, count: tabs.length, activeTabId: session.activeTabId, lastAgentUpdate: session._lastAgentUpdate || null, tabs }) });           return;         } @@ -3079,6 +3199,12 @@ const server = http.createServer(async (req, res) => {           }           const session = sessions.get(sessionId);           try {+            // v1.8.23: BLUR the page's active element BEFORE raising. Raising steals the OS+            // focus, and a keystroke the user was mid-typing can land in whatever input the+            // pup page had focused (a form field became "John Lauer a"). With no focused+            // element, a stray key has nowhere to land. (The residual SetForegroundWindow+            // input-queue timing is AD-owned when AD intercepts this verb — flagged upstream.)+            try { await session.page.evaluate(() => { const a = document.activeElement; if (a && a.blur) a.blur(); }); } catch (e) {}             // First activate the tab within Chrome             await session.page.bringToFront();             activeSessionId = sessionId;