app
Pup - the AI's own browser
Public Made by Adomby adom
pup gives your AI a real browser on your desktop that it drives completely, in the background, signed in as you: windows that never jump in front of your work, one shared profile that learns your logins, and every window labeled on the taskbar with its AI thread's name.
master
John Lauer
v2.0.579 (insiders): ext lane at input parity: click, hover, type, press_key, scroll, fill, element and full-res screen…
418ec84
1mo ago
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195
// adom-native-browser :: CDP helpers (chrome.debugger)
// --------------------------------------------------------
// Used ONLY for verbs that need DevTools Protocol parity with pup: full-page screenshots and
// arbitrary expression eval. Navigation/tabs/windows go through native chrome.* APIs (native.js) so
// they never raise the "started debugging this browser" banner.
//
// Banner policy: attach lazily on first CDP use of a tab, auto-detach after IDLE_MS of no CDP use, so
// the banner only shows while actively driving. On Adom-managed machines the browser is launched with
// --silent-debugger-extension-api, which removes it entirely.
//
// DEBOUNCE (John, 2026-07-14): the banner slides IN on attach and OUT on detach, and each transition
// REFLOWS the page (content shifts up/down), which moves where a click lands and ruins the user's own
// clicks in a window they're looking at. A short idle window made it flicker in/out between every verb
// burst (think/page-load pauses > 8s detached it, then the next verb re-attached). So IDLE_MS is long
// enough to span normal per-verb gaps: the banner stays STEADY through a drive session and only slides
// out once the AI has genuinely stopped for a while (not left up forever - screen real estate matters).
// We also NEVER attach to a tab the caller didn't explicitly target (see resolveTabId requireTarget), so
// the banner can only ever appear on a window the AI is actually driving, not the user's active tab.
const PROTO = '1.3';
const IDLE_MS = 45000; // was 8000 — long enough to not flicker between verbs, short enough to not linger
const attached = new Map(); // tabId -> idleTimer
function debuggee(tabId) { return { tabId }; }
function send(tabId, method, params) {
return new Promise((resolve, reject) => {
chrome.debugger.sendCommand(debuggee(tabId), method, params || {}, (res) => {
const err = chrome.runtime.lastError;
if (err) reject(new Error(err.message)); else resolve(res);
});
});
}
function bumpIdle(tabId) {
const t = attached.get(tabId);
if (t) clearTimeout(t);
attached.set(tabId, setTimeout(() => detach(tabId).catch(() => {}), IDLE_MS));
}
async function attach(tabId) {
if (attached.has(tabId)) { bumpIdle(tabId); return; }
await new Promise((resolve, reject) => {
chrome.debugger.attach(debuggee(tabId), PROTO, () => {
const err = chrome.runtime.lastError;
if (err && !/already attached/i.test(err.message)) reject(new Error(err.message));
else resolve();
});
});
attached.set(tabId, null);
await send(tabId, 'Page.enable').catch(() => {});
await send(tabId, 'Runtime.enable').catch(() => {});
await send(tabId, 'Log.enable').catch(() => {});
// CRITICAL for BACKGROUND clicking: a window fully occluded by another (e.g. the user's maximized
// window over our background one) is marked hidden by Chrome's native-occlusion tracking, so
// Input.dispatchMouseEvent stops hit-testing and clicks silently land NOWHERE (eval/screenshot still
// work - they don't need a hit-testable frame). Emulating focus makes the page behave as focused+visible
// so isTrusted CDP input lands, WITHOUT ever raising the window (stays background, no focus theft).
await send(tabId, 'Emulation.setFocusEmulationEnabled', { enabled: true }).catch(() => {});
bumpIdle(tabId);
}
// pup (2.0.578) drives its extension-lane windows through THIS driver instead of a second one: one raw
// DevTools call on nbe's own attach + idle-detach lifecycle. Additive; nothing else here changes.
export async function raw(tabId, method, params) {
await attach(tabId);
const r = await send(tabId, method, params || {});
bumpIdle(tabId);
return r;
}
export async function detach(tabId) {
const t = attached.get(tabId);
if (t) clearTimeout(t);
if (!attached.has(tabId)) return;
attached.delete(tabId);
await new Promise((resolve) => chrome.debugger.detach(debuggee(tabId), () => { void chrome.runtime.lastError; resolve(); }));
}
// ---- BACKGROUND CAPTURE: force a frame, never a foreground -----------------
// (John, 2026-08-22) The excuse an AI reaches for when a background capture comes back blank is "I had
// to bring the window forward to get a good screenshot." That is the single rudest thing nb can cause,
// and it is ALSO unnecessary - it is a compositor problem, not a focus problem. Chrome stops painting a
// window it believes is occluded (CalculateNativeWinOcclusion) and FREEZES a backgrounded page's
// rendering, so Page.captureScreenshot can hand back a stale or uniform frame. The answer is to make the
// renderer produce a frame, which we can do without touching the user's focus:
// 1. Page.setWebLifecycleState{active} - the documented CDP un-freeze for a backgrounded page
// 2. a double-rAF round-trip in the page - forces the renderer to commit one fresh frame
// 3. re-capture, and only judge the result after that
// Best-effort at every step; a window that never needed it pays two animation frames. If we ever CANNOT
// get a live frame we still refuse to foreground - we say so, and hand the AI the WGC fallback, which
// captures an occluded window at the OS level.
async function forceFrame(tabId) {
const used = [];
try { await send(tabId, 'Page.setWebLifecycleState', { state: 'active' }); used.push('lifecycle:active'); } catch {}
try {
await send(tabId, 'Runtime.evaluate', {
expression: 'new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(()=>r(1))))',
awaitPromise: true, returnByValue: true,
});
used.push('raf');
} catch {}
return used;
}
// A capture that is one flat colour is the occlusion signature (Chrome handed us an unpainted surface).
// Sampled at <=64x64 so the check costs microseconds on a full-page shot.
async function frameIsBlank(b64) {
try {
const bmp = await bitmapFromB64(b64);
const w = Math.max(1, Math.min(bmp.width, 64)), h = Math.max(1, Math.min(bmp.height, 64));
const canvas = new OffscreenCanvas(w, h);
const g = canvas.getContext('2d', { willReadFrequently: true });
g.drawImage(bmp, 0, 0, w, h);
bmp.close && bmp.close();
const d = g.getImageData(0, 0, w, h).data;
let min = 255, max = 0;
for (let i = 0; i < d.length; i += 4) {
const v = (d[i] + d[i + 1] + d[i + 2]) / 3;
if (v < min) min = v;
if (v > max) max = v;
}
return (max - min) < 3;
} catch { return false; }
}
// ---- nbrowser_screenshot / nbrowser_screenshot_full_res --------------------
export async function screenshot(tabId, { fullPage = false, maxWidth = 1568, resize = true } = {}) {
await attach(tabId);
const forced = await forceFrame(tabId);
const params = { format: 'png', captureBeyondViewport: !!fullPage };
if (fullPage) {
const m = await send(tabId, 'Page.getLayoutMetrics');
const css = m.cssContentSize || m.contentSize;
params.clip = { x: 0, y: 0, width: Math.ceil(css.width), height: Math.ceil(css.height), scale: 1 };
}
let res = await send(tabId, 'Page.captureScreenshot', params);
// Blank frame = the compositor never painted (occluded window). Re-force and re-capture ONCE. We do
// NOT foreground to fix this, ever - that is the user's focus, not ours to take.
let recovered = false, stillBlank = false;
if (await frameIsBlank(res.data)) {
forced.push(...(await forceFrame(tabId)), 'recapture');
const res2 = await send(tabId, 'Page.captureScreenshot', params);
if (await frameIsBlank(res2.data)) stillBlank = true;
else { res = res2; recovered = true; }
}
bumpIdle(tabId);
const out = resize ? await resizePng(res.data, maxWidth) : await pngDims(res.data);
const iw = out.width, ih = out.height, ow = out.origWidth ?? out.width, oh = out.origHeight ?? out.height;
// Viewport coordinate-map, sampled AT capture time, so screenshot-pixel -> click-coordinate mapping is
// exact and never needs a separate probe (and can't drift from a later scroll). To turn a pixel (sx,sy)
// in the RETURNED image into a viewport click coordinate (CSS px, what nbrowser_click/input_dispatch
// take): clickX = sx * cssPerImageX, clickY = sy * cssPerImageY. scrollX/scrollY are included for
// page-absolute math; devicePixelRatio + captured/image dims are given so any custom mapping is possible.
let viewport = null;
try {
const m = await evaluate(tabId, '({iw:innerWidth,ih:innerHeight,dpr:devicePixelRatio,sx:Math.round(scrollX),sy:Math.round(scrollY),dw:Math.round(document.documentElement.scrollWidth),dh:Math.round(document.documentElement.scrollHeight)})');
if (m && typeof m === 'object') {
const spanW = fullPage ? m.dw : m.iw, spanH = fullPage ? m.dh : m.ih; // CSS px the image spans
viewport = {
innerWidth: m.iw, innerHeight: m.ih, devicePixelRatio: m.dpr,
scrollX: m.sx, scrollY: m.sy,
capturedWidth: ow, capturedHeight: oh, // raw CDP capture px (~ span*dpr)
imageWidth: iw, imageHeight: ih, // the returned (possibly downscaled) image px
resizeRatio: ow ? +(iw / ow).toFixed(6) : 1,
fullPage: !!fullPage, // true = image spans the whole page; coords are page-absolute
cssPerImageX: iw ? +(spanW / iw).toFixed(6) : 1, // <- multiply a screenshot X by this to get a click X (CSS px)
cssPerImageY: ih ? +(spanH / ih).toFixed(6) : 1,
};
}
} catch (e) {}
const shot = { ok: true, base64: out.base64, width: iw, height: ih, origWidth: ow, origHeight: oh,
resized: !!out.resized, sizeKB: Math.round((out.base64.length * 3 / 4) / 1024), viewport,
// Tell the AI what we did to get a live frame WITHOUT focus, so "I had to foreground it to see the
// page" is never a conclusion it can honestly reach.
capture: { forcedRepaint: forced, blankRecovered: recovered, blank: stillBlank } };
if (stillBlank) {
shot._hint = 'The page composited a BLANK frame even after a forced repaint (lifecycle:active + double rAF). ' +
'Do NOT foreground the window to fix this - that steals the user\'s focus and is not the fix anyway. ' +
'Use ab desktop_screenshot_window {hwnd} instead: WGC captures an OCCLUDED window at the OS level, ' +
'chrome included. If you own this Chrome, relaunching it with --disable-features=CalculateNativeWinOcclusion ' +
'--disable-backgrounding-occluded-windows --disable-renderer-backgrounding stops it happening at all.';
} else if (recovered) {
shot._hint = 'This window was NOT painting (occluded/backgrounded); nbe forced a repaint and captured a live ' +
'frame with the window still in the background. You never need to foreground a window to screenshot it.';
}
// A native JS dialog does NOT appear in a CDP capture (it's browser chrome, not page content), so tell
// the AI explicitly - otherwise it screenshots, sees a normal page, and never realizes a modal is up.
const dlg = _openDialogs.get(Number(tabId));
if (dlg) shot._modal = `⚠️ a native ${dlg.type} dialog is open ("${(dlg.message || '').slice(0, 120)}") - it is NOT in this screenshot (dialogs are browser chrome) and it is BLOCKING the page. Dismiss it with nbrowser_handle_dialog {sessionId, accept:true|false} before doing anything else.`;
return shot;
}
// ---- nbrowser_eval / nbrowser_evaluate ------------------------------------
export async function evaluate(tabId, expr) {
await attach(tabId);
// SPA route changes (e.g. Gusto's wizard steps) tear down the execution
// context; for a short window Runtime.evaluate can throw "Cannot find
// context with specified id" or come back with neither result nor
// exceptionDetails. Retry a few times so callers don't see a phantom
// empty/undefined right after an in-app navigation.
let res;
for (let attempt = 0; attempt < 4; attempt++) {
try {
res = await send(tabId, 'Runtime.evaluate', {
expression: `(${expr})`,
returnByValue: true,
awaitPromise: true,
userGesture: true,
});
} catch (err) {
const msg = String((err && err.message) || err);
if (/context|detached|target closed|destroyed/i.test(msg) && attempt < 3) {
await attach(tabId); // re-resolve the context
await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
continue;
}
throw err;
}
// A torn-down context yields an empty envelope (no result, no exception).
if (!res || (!res.result && !res.exceptionDetails)) {
if (attempt < 3) {
await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
continue;
}
}
break;
}
bumpIdle(tabId);
if (res && res.exceptionDetails) {
const e = res.exceptionDetails;
return { __error: (e.exception && (e.exception.description || e.exception.value)) || e.text };
}
return res && res.result ? res.result.value : undefined;
}
// ---- input dispatch (CDP Input domain → isTrusted events) ------------------
const KEYMAP = {
Enter: { key: 'Enter', code: 'Enter', vk: 13 },
Tab: { key: 'Tab', code: 'Tab', vk: 9 },
Escape: { key: 'Escape', code: 'Escape', vk: 27 },
Backspace: { key: 'Backspace', code: 'Backspace', vk: 8 },
Delete: { key: 'Delete', code: 'Delete', vk: 46 },
ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 },
ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 },
ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 },
ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 },
Home: { key: 'Home', code: 'Home', vk: 36 },
End: { key: 'End', code: 'End', vk: 35 },
};
async function pressKey(tabId, name) {
const k = KEYMAP[name] || { key: name, code: name, vk: name.length === 1 ? name.toUpperCase().charCodeAt(0) : 0 };
const base = { key: k.key, code: k.code, windowsVirtualKeyCode: k.vk, nativeVirtualKeyCode: k.vk };
await send(tabId, 'Input.dispatchKeyEvent', { type: 'keyDown', ...base });
await send(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', ...base });
}
async function mouseClick(tabId, x, y, button = 'left', clickCount = 1) {
await send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y });
await send(tabId, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount, buttons: button === 'right' ? 2 : 1 });
await send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount, buttons: 0 });
}
// Port of pup's modal-aware smart-pick: resolve a selector to the best visible element's center.
function scoreExpr(selector, firstMatch) {
const sel = JSON.stringify(selector);
return `(function(){
var els=[].slice.call(document.querySelectorAll(${sel}));
if(!els.length)return{found:false,count:0};
function vis(el){var r=el.getBoundingClientRect(),s=getComputedStyle(el);
return r.width>0&&r.height>0&&s.visibility!=='hidden'&&s.display!=='none'&&parseFloat(s.opacity||'1')>0.01;}
var c=els.filter(vis); if(!c.length)c=els;
var modal=document.querySelector('dialog[open],[aria-modal="true"],[role="dialog"]');
if(modal){var m=c.filter(function(e){return modal.contains(e);});if(m.length)c=m;}
var chosen=${firstMatch ? 'c[0]' : 'c.reduce(function(a,b){var ra=a.getBoundingClientRect(),rb=b.getBoundingClientRect();return (rb.width*rb.height)>(ra.width*ra.height)?b:a;})'};
if(chosen.scrollIntoView)chosen.scrollIntoView({block:'center',inline:'center'});
var r=chosen.getBoundingClientRect();
return{found:true,count:c.length,x:Math.round(r.left+r.width/2),y:Math.round(r.top+r.height/2),
rect:{x:Math.round(r.left),y:Math.round(r.top),w:Math.round(r.width),h:Math.round(r.height)},
text:((chosen.innerText||chosen.value||'')+'').slice(0,60),modal:!!modal};
})()`;
}
function focusExpr(selector) {
const sel = JSON.stringify(selector);
return `(function(){var e=document.querySelector(${sel});if(!e)return{found:false};if(e.focus)e.focus();if(e.scrollIntoView)e.scrollIntoView({block:'center'});return{found:true};})()`;
}
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// ---- visible Adom cursor — EXACT match to Hydrogen's AI cursor (hd-control AI_CURSOR_JS) --------------
// Teal Adom arrow (#00b8b0) + dark-teal outline + teal glow, a pulsing teal ring, and a monospace
// "Adom" label so the user always knows it's an Adom-driven cursor (trust). Kept byte-faithful to Hydrogen
// so the cursor looks identical across the whole product.
function cursorScript(action, x, y, label) {
var id = '__adom_cursor__';
var c = document.getElementById(id);
if (action === 'hide') { if (c) c.style.display = 'none'; return; }
if (!c) {
c = document.createElement('div');
c.id = id;
c.style.cssText = 'position:fixed;z-index:2147483647;pointer-events:none;transition:left 0.15s,top 0.15s;display:none;';
var g = document.createElement('div'); // pulsing teal glow ring
g.id = '__adom_cursor_ring__';
g.style.cssText = 'position:absolute;top:-16px;left:-16px;width:40px;height:40px;border-radius:50%;background:rgba(0,184,176,0.15);border:2px solid rgba(0,184,176,0.4);animation:adomCurP 1.5s ease-in-out infinite;';
c.appendChild(g);
var a = document.createElement('div'); // the teal Adom arrow (Hydrogen's exact SVG)
a.innerHTML = '<svg width="20" height="28" viewBox="0 0 20 28" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L1 22L6.5 16.5L10.5 26L14 24.5L10 15L17 15Z" fill="#00b8b0" stroke="#004d49" stroke-width="1.2" stroke-linejoin="round"/></svg>';
a.style.cssText = 'position:absolute;top:0;left:0;filter:drop-shadow(0 0 6px rgba(0,184,176,0.7));';
c.appendChild(a);
var l = document.createElement('div'); // the "Adom" trust label
l.id = '__adom_cursor_label__';
l.style.cssText = 'position:absolute;top:32px;left:8px;white-space:nowrap;font-size:11px;color:#00b8b0;background:rgba(13,17,23,0.92);padding:2px 10px;border-radius:4px;border:1px solid rgba(0,184,176,0.6);font-family:monospace;';
c.appendChild(l);
(document.body || document.documentElement).appendChild(c);
if (!document.getElementById('__adom_cur_kf__')) {
var s = document.createElement('style'); s.id = '__adom_cur_kf__';
s.textContent = '@keyframes adomCurP{0%{transform:scale(1);opacity:0.7}50%{transform:scale(1.5);opacity:0.2}100%{transform:scale(1);opacity:0.7}}@keyframes adomCurClick{0%{transform:scale(1);opacity:0.85}45%{transform:scale(2.2);opacity:0.05}100%{transform:scale(1);opacity:0.7}}';
document.head.appendChild(s);
}
}
var lab = c.querySelector('#__adom_cursor_label__');
if (lab) {
lab.textContent = (label != null ? label : 'Adom');
// flip the label to the LEFT of the cursor when near the right edge, so it never clips offscreen
var nearRight = (x != null) && (x > (window.innerWidth - 250));
if (nearRight) { lab.style.left = 'auto'; lab.style.right = '0px'; lab.style.textAlign = 'right'; }
else { lab.style.left = '8px'; lab.style.right = 'auto'; lab.style.textAlign = 'left'; }
}
if (x != null && y != null) { c.style.left = x + 'px'; c.style.top = y + 'px'; }
c.style.display = 'block';
if (action === 'click') {
var ring = c.querySelector('#__adom_cursor_ring__');
if (ring) { ring.style.animation = 'none'; void ring.offsetWidth; ring.style.animation = 'adomCurClick 0.45s ease-out, adomCurP 1.5s ease-in-out infinite 0.45s'; }
}
}
export async function cursor(tabId, action, x, y, label) {
try { await chrome.scripting.executeScript({ target: { tabId }, func: cursorScript, args: [action, x == null ? null : x, y == null ? null : y, label == null ? null : String(label)] }); } catch (e) {}
}
// Hydrogen-style intention narration for the cursor label. phase 'find' = while gliding there; 'act' = doing it.
function phaseLabel(type, tgt, phase) {
const name = tgt ? `"${tgt}"` : 'this';
if (type === 'hover') return phase === 'act' ? `Hovering ${name}` : `Moving to ${name}`;
if (type === 'dblclick' || type === 'doubleclick') return phase === 'act' ? `Double-clicking ${name}` : `Finding ${name}`;
if (type === 'rightclick' || type === 'contextmenu') return phase === 'act' ? `Right-clicking ${name}` : `Finding ${name}`;
return phase === 'act' ? `Clicking ${name}` : `Finding ${name}`;
}
// ---- smart finder: locate visible interactive elements (for the screenshot->find->click loop) -----
// Injected via chrome.scripting (no debugger banner). Matches by visible text / aria-label.
// `rich`: when true, attach per-candidate debug data (score/scoreBreakdown, occlusion, visibility,
// inViewport, a best-effort CSS selector) WITHOUT changing the selection order — chosen stays index 0,
// so {debug:true} never regresses a click. The numeric `score` is for human/AI debugging only.
function findFn(query, limit, rich) {
var q = (query || '').toLowerCase();
var sel = 'a,button,[role=button],[role=link],[role=menuitem],[role=menuitemradio],[role=menuitemcheckbox],[role=tab],[role=checkbox],[role=radio],[role=switch],[role=option],[role=combobox],[role=treeitem],[role=gridcell],input,select,textarea,summary,[onclick],[tabindex],[contenteditable]';
// Deep collector: light DOM + every OPEN shadow root (recursively). Custom-element finance/SPA
// widgets (Schwab account picker, QBO report search) put their real options INSIDE shadow roots that
// document.querySelectorAll can't see - so we descend into each element's .shadowRoot. (Closed shadow
// roots are invisible to any script; only open ones are reachable, which covers the vast majority.)
function deepQ(root, s) {
var acc = [];
(function walk(node) {
try { acc.push.apply(acc, node.querySelectorAll(s)); } catch (e) {}
var kids; try { kids = node.querySelectorAll('*'); } catch (e) { kids = []; }
for (var i = 0; i < kids.length; i++) { var sr = kids[i].shadowRoot; if (sr) walk(sr); }
})(root || document);
return acc;
}
// Scroll the winner to centre and re-read its geometry: the coordinates we hand back are viewport
// coordinates, so a candidate that was off-screen when matched must be brought on-screen BEFORE its
// x/y are computed. Without this the dispatch fires at stale/offscreen coordinates and silently
// misses (nb-bridge #20: right element resolved, changed:false, DOM untouched).
function centre(list) {
if (!list.length) return list;
var top = list[0];
if (top && top._el && top._el.scrollIntoView) {
try { top._el.scrollIntoView({ block: 'center', inline: 'center' }); } catch (e) {}
var r = top._el.getBoundingClientRect();
top.rect = { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height) };
top.x = Math.round(r.left + r.width / 2); top.y = Math.round(r.top + r.height / 2);
}
for (var i = 0; i < list.length; i++) delete list[i]._el;
return list;
}
var els = deepQ(document, sel);
// RENDERED, not necessarily ON-SCREEN. Requiring the element to be inside the viewport made every
// target the user had scrolled past invisible to click-by-text (nb-bridge #20): the interactive pass
// found nothing and the fallback returned <body>/<html>. resolveCoords scrolls the chosen element to
// centre before dispatching, so off-viewport-but-rendered is perfectly clickable.
function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01; }
// NEVER a click target: the document/root containers match almost any text and clicking them does
// nothing (or hits whatever is underneath). Excluding them also removes the phantom second
// "candidate" that produced false ambiguous_target refusals.
function clickable(el) { var t = (el.tagName || '').toUpperCase(); return t !== 'HTML' && t !== 'BODY' && t !== 'HEAD' && t !== 'SCRIPT' && t !== 'STYLE' && t !== 'NOSCRIPT'; }
function txt(el) { return ((el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.getAttribute('title') || '') + '').replace(/\s+/g, ' ').trim(); }
function cssPath(el) { try { if (!el || el.nodeType !== 1) return ''; if (el.id) return '#' + el.id; var p = el.tagName.toLowerCase(); var cn = el.className; cn = (cn && cn.baseVal !== undefined ? cn.baseVal : cn) || ''; var cls = ('' + cn).trim().split(/\s+/).filter(Boolean).slice(0, 2); if (cls.length) p += '.' + cls.join('.'); var par = el.parentElement; if (par) { var same = [].slice.call(par.children).filter(function (x) { return x.tagName === el.tagName; }); if (same.length > 1) p += ':nth-of-type(' + (same.indexOf(el) + 1) + ')'; } return p; } catch (e) { return ''; } }
function richOf(el, t, al) {
var r = el.getBoundingClientRect(); var tl = (t || '').toLowerCase(); var tag = el.tagName.toLowerCase();
var textMatch = !q ? 0 : tl === q ? 100 : al === q ? 95 : tl.indexOf(q) === 0 ? 70 : tl.indexOf(q) >= 0 ? 50 : al.indexOf(q) >= 0 ? 40 : 0;
var clickableTag = (/^(a|button|input|select|textarea|summary)$/.test(tag) || el.getAttribute('role')) ? 20 : 0;
var visible = vis(el); var visibility = visible ? 10 : 0;
var inViewport = r.top >= 0 && r.left >= 0 && r.bottom <= (innerHeight || 0) && r.right <= (innerWidth || 0);
var cx = Math.round(r.left + r.width / 2), cy = Math.round(r.top + r.height / 2);
var occluded = false, occludedBy = '';
try { var top = document.elementFromPoint(cx, cy); if (top && top !== el && !el.contains(top) && !top.contains(el)) { occluded = true; occludedBy = cssPath(top); } } catch (e) {}
var occlusionPenalty = occluded ? -30 : 0;
return { score: textMatch + clickableTag + visibility + occlusionPenalty, scoreBreakdown: { textMatch: textMatch, clickableTag: clickableTag, visibility: visibility, occlusionPenalty: occlusionPenalty, area: Math.round(r.width * r.height) }, visible: visible, occluded: occluded, occludedBy: occludedBy, inViewport: inViewport, selector: cssPath(el) };
}
function mk(el, t, r) { var o = { _el: el, text: (t || '').slice(0, 80), tag: el.tagName.toLowerCase(), role: el.getAttribute('role') || '', type: el.type || '', x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), rect: { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height), centerX: Math.round(r.left + r.width / 2), centerY: Math.round(r.top + r.height / 2) } }; if (rich) { var rf = richOf(el, t, (el.getAttribute('aria-label') || '').toLowerCase()); for (var k in rf) o[k] = rf[k]; } return o; }
var out = [];
for (var i = 0; i < els.length; i++) {
var el = els[i]; if (!vis(el) || !clickable(el)) continue; var t = txt(el); var al = (el.getAttribute('aria-label') || '').toLowerCase();
if (q && t.toLowerCase().indexOf(q) < 0 && al.indexOf(q) < 0) continue;
out.push(mk(el, t, el.getBoundingClientRect()));
}
// Fallback: hover/move targets are often NON-interactive (info icons, cards, tooltip
// triggers, plain spans). If nothing interactive matched the text, scan all elements and
// take the smallest (leaf-most) visible element whose text contains the query — that's the
// specific thing the user named, not a giant wrapping container.
if (q && out.length === 0) {
var all = deepQ(document, '*');
var fb = [];
for (var j = 0; j < all.length; j++) {
var e2 = all[j]; if (!vis(e2) || !clickable(e2)) continue;
var t2 = ((e2.innerText || e2.textContent || '') + '').replace(/\s+/g, ' ').trim();
if (!t2 || t2.toLowerCase().indexOf(q) < 0) continue;
var r2 = e2.getBoundingClientRect();
var o2 = mk(e2, t2, r2); o2.area = r2.width * r2.height; fb.push(o2);
}
// exact text match wins, else smallest area (most specific leaf) — return directly,
// skipping the interactive-pass sort below (which prefers LARGEST, wrong for leaves).
fb.sort(function (a, b) { var ae = a.text.toLowerCase() === q ? 0 : 1, be = b.text.toLowerCase() === q ? 0 : 1; if (ae !== be) return ae - be; return a.area - b.area; });
return centre(fb.map(function (o) { delete o.area; return o; }).slice(0, limit || 40));
}
out.sort(function (a, b) { var ae = a.text.toLowerCase() === q ? 0 : 1, be = b.text.toLowerCase() === q ? 0 : 1; if (ae !== be) return ae - be; return (b.rect.w * b.rect.h) - (a.rect.w * a.rect.h); });
return centre(out.slice(0, limit || 40));
}
export async function find(tabId, query, limit, rich) {
try {
const r = await chrome.scripting.executeScript({ target: { tabId }, func: findFn, args: [query || '', limit || 40, !!rich] });
const els = (r && r[0] && r[0].result) || [];
return { ok: true, query: query || '', count: els.length, elements: els };
} catch (e) { return { ok: false, error: String((e && e.message) || e) }; }
}
// All visible matches of a CSS selector, with debug data — for {debug}/{resolveOnly} on selector targets.
function selectorCandidatesFn(selector) {
// Light-DOM first (fast, honors combinators). If it finds nothing, deep-walk every open shadow root and
// match each element with .matches(selector) - so a stable attribute (e.g. [data-testid=x]) or class/tag
// selector pierces custom-element shadow DOM. (Cross-shadow combinators like ".a > .b" only match within
// one root; simple attribute/class/tag/role selectors are what this recovers, which is the common case.)
var els = [].slice.call(document.querySelectorAll(selector));
if (!els.length) {
var deep = [];
(function walk(node) {
var kids; try { kids = node.querySelectorAll('*'); } catch (e) { kids = []; }
for (var i = 0; i < kids.length; i++) {
var el = kids[i];
try { if (el.matches && el.matches(selector)) deep.push(el); } catch (e) {}
if (el.shadowRoot) walk(el.shadowRoot);
}
})(document);
els = deep;
}
function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01; }
function cssPath(el) { try { if (el.id) return '#' + el.id; var p = el.tagName.toLowerCase(); var par = el.parentElement; if (par) { var same = [].slice.call(par.children).filter(function (x) { return x.tagName === el.tagName; }); if (same.length > 1) p += ':nth-of-type(' + (same.indexOf(el) + 1) + ')'; } return p; } catch (e) { return ''; } }
var c = els.filter(vis); if (!c.length) c = els;
var modal = document.querySelector('dialog[open],[aria-modal="true"],[role="dialog"]');
if (modal) { var m = c.filter(function (e) { return modal.contains(e); }); if (m.length) c = m; }
if (!c.length) return [];
var chosenEl = c.reduce(function (a, b) { var ra = a.getBoundingClientRect(), rb = b.getBoundingClientRect(); return (rb.width * rb.height) > (ra.width * ra.height) ? b : a; });
return c.slice(0, 40).map(function (el) {
var r = el.getBoundingClientRect(); var cx = Math.round(r.left + r.width / 2), cy = Math.round(r.top + r.height / 2);
var occ = false, occBy = ''; try { var top = document.elementFromPoint(cx, cy); if (top && top !== el && !el.contains(top) && !top.contains(el)) { occ = true; occBy = cssPath(top); } } catch (e) {}
return { text: ((el.innerText || el.value || '') + '').replace(/\s+/g, ' ').trim().slice(0, 80), tag: el.tagName.toLowerCase(), role: el.getAttribute('role') || '', selector: cssPath(el), x: cx, y: cy, rect: { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height), centerX: cx, centerY: cy }, score: el === chosenEl ? 100 : 50, scoreBreakdown: { rule: 'largest-visible-match', area: Math.round(r.width * r.height) }, visible: vis(el), occluded: occ, occludedBy: occBy, inViewport: r.top >= 0 && r.left >= 0 && r.bottom <= innerHeight && r.right <= innerWidth, chosen: el === chosenEl };
});
}
// ---- nbrowser_select: open a custom dropdown and pick an option by its VISIBLE TEXT ----------------
// Solves: coordinate clicks on virtualized/custom option lists land on the WRONG row (mis-selecting a
// bank account by one row). This locates the trigger (by accessible `label` or the `near` text beside
// it), opens it with an isTrusted CDP click, WAITS for the options to render, clicks the EXACT text
// match (shadow-piercing), and returns the newly-displayed value so the caller can verify.
function selectTriggerFn(label, near) {
function deepQ(root, s) { var a = []; (function w(n) { try { a.push.apply(a, n.querySelectorAll(s)); } catch (e) {} var k; try { k = n.querySelectorAll('*'); } catch (e) { k = []; } for (var i = 0; i < k.length; i++) { var sr = k[i].shadowRoot; if (sr) w(sr); } })(root || document); return a; }
function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01 && r.top < innerHeight && r.bottom > 0; }
function txt(el) { return (((el.getAttribute && (el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('placeholder'))) || el.innerText || el.textContent || '') + '').replace(/\s+/g, ' ').trim(); }
var cand = null;
if (label) {
var lab = (label + '').toLowerCase();
var trigs = deepQ(document, 'select,[role=combobox],[role=listbox],[aria-haspopup],button,[role=button],input').filter(vis);
trigs.sort(function (a, b) { function sc(e) { var t = txt(e).toLowerCase(); return t === lab ? 3 : t.indexOf(lab) === 0 ? 2 : t.indexOf(lab) >= 0 ? 1 : 0; } return sc(b) - sc(a); });
if (trigs.length && txt(trigs[0]).toLowerCase().indexOf(lab) >= 0) cand = trigs[0];
}
if (!cand && near) {
var nl = (near + '').toLowerCase();
var texts = deepQ(document, 'label,span,div,legend,th,td,p').filter(function (e) { return vis(e) && txt(e).toLowerCase().indexOf(nl) >= 0; });
texts.sort(function (a, b) { return txt(a).length - txt(b).length; });
if (texts.length) {
var L = texts[0], forId = L.getAttribute && L.getAttribute('for');
if (forId) { var t = document.getElementById(forId); if (t && vis(t)) cand = t; }
if (!cand) {
var lr = L.getBoundingClientRect(), lx = lr.left + lr.width / 2, ly = lr.top + lr.height / 2;
var ctrls = deepQ(document, 'select,[role=combobox],[aria-haspopup],button,[role=button],input,[role=listbox]').filter(vis);
ctrls.sort(function (a, b) { var ra = a.getBoundingClientRect(), rb = b.getBoundingClientRect(); return Math.hypot(ra.left + ra.width / 2 - lx, ra.top + ra.height / 2 - ly) - Math.hypot(rb.left + rb.width / 2 - lx, rb.top + rb.height / 2 - ly); });
if (ctrls.length) cand = ctrls[0];
}
}
}
if (!cand) return { found: false };
var r = cand.getBoundingClientRect();
// `display` = the trigger's now-VISIBLE text (what a human sees selected), preferred for the readback.
// `current` = its accessible name (aria-label/title/placeholder, used for finding) - kept as fallback.
var disp = ((cand.innerText || cand.textContent || '') + '').replace(/\s+/g, ' ').trim();
return { found: true, x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), tag: cand.tagName.toLowerCase(), isNativeSelect: cand.tagName.toLowerCase() === 'select', current: txt(cand).slice(0, 120), display: disp.slice(0, 120) };
}
function nativeSetFn(x, y, optionText) {
var el = document.elementFromPoint(x, y); while (el && el.tagName !== 'SELECT') el = el.parentElement;
if (!el) return { ok: false, error: 'native select not found at point' };
var ot = (optionText + '').toLowerCase(), opts = [].slice.call(el.options), m = null;
for (var i = 0; i < opts.length; i++) { if ((opts[i].text || '').replace(/\s+/g, ' ').trim().toLowerCase() === ot) { m = opts[i]; break; } }
if (!m) for (var j = 0; j < opts.length; j++) { if ((opts[j].text || '').toLowerCase().indexOf(ot) >= 0) { m = opts[j]; break; } }
if (!m) return { ok: false, error: 'option not found in select: ' + optionText };
el.value = m.value; el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true }));
return { ok: true, selected: (m.text || '').replace(/\s+/g, ' ').trim() };
}
function findOptionFn(optionText) {
function deepQ(root, s) { var a = []; (function w(n) { try { a.push.apply(a, n.querySelectorAll(s)); } catch (e) {} var k; try { k = n.querySelectorAll('*'); } catch (e) { k = []; } for (var i = 0; i < k.length; i++) { var sr = k[i].shadowRoot; if (sr) w(sr); } })(root || document); return a; }
function txt(e) { return ((e.innerText || e.textContent || '') + '').replace(/\s+/g, ' ').trim(); }
var ot = (optionText + '').toLowerCase();
var opts = deepQ(document, '[role=option],[role=menuitem],[role=menuitemradio],[role=treeitem],[role=row],li,option,a,button').filter(function (e) { var r = e.getBoundingClientRect(), s = getComputedStyle(e); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01; });
var exact = opts.filter(function (e) { return txt(e).toLowerCase() === ot; });
var cont = exact.length ? exact : opts.filter(function (e) { return txt(e).toLowerCase().indexOf(ot) >= 0; });
if (!cont.length) return { found: false, optionCount: opts.length };
cont.sort(function (a, b) { var ra = a.getBoundingClientRect(), rb = b.getBoundingClientRect(); return (ra.width * ra.height) - (rb.width * rb.height); }); // smallest = leaf-most option row
var el = cont[0], r = el.getBoundingClientRect();
return { found: true, x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), text: txt(el).slice(0, 120), exact: exact.length > 0, optionCount: opts.length };
}
export async function selectOption(tabId, args) {
await attach(tabId);
const timeoutMs = Math.min(Math.max(Number(args.timeoutMs) || 4000, 500), 20000);
const inj = (func, a) => chrome.scripting.executeScript({ target: { tabId }, func, args: a }).then((r) => r && r[0] && r[0].result);
const t = await inj(selectTriggerFn, [args.label || '', args.near || '']);
if (!t || !t.found) return { ok: false, error: 'dropdown trigger not found' + (args.label ? ` for label "${args.label}"` : '') + (args.near ? ` near "${args.near}"` : ''), _hint: 'pass label (the control\'s accessible name / visible text) or near (a visible text label beside it)' };
if (t.isNativeSelect) {
const r = await inj(nativeSetFn, [t.x, t.y, args.optionText || '']);
return r && r.ok ? { ok: true, selected: r.selected, via: 'native-select', matchedExact: true, _hint: 'set a native <select> by option text and dispatched change.' } : { ok: false, error: (r && r.error) || 'native select failed' };
}
await mouseClick(tabId, t.x, t.y); // open (isTrusted)
let opt = null; const deadline = Date.now() + timeoutMs; let lastCount = 0;
while (Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 150));
const o = await inj(findOptionFn, [args.optionText || '']);
if (o) { lastCount = o.optionCount || lastCount; if (o.found) { opt = o; break; } }
}
if (!opt) return { ok: false, error: `option "${args.optionText}" did not appear after opening the dropdown (${lastCount} option(s) seen)`, _hint: 'the trigger opened but no option matched that EXACT visible text; check spelling/case, or raise timeoutMs for a slow/virtualized list' };
await mouseClick(tabId, opt.x, opt.y); // pick (isTrusted)
await new Promise((r) => setTimeout(r, 220));
const after = await inj(selectTriggerFn, [args.label || '', args.near || '']);
// Prefer the trigger's now-visible display text; fall back to the option we clicked.
const selected = (after && after.found && (after.display || after.current)) || opt.text;
return { ok: true, selected, matchedExact: opt.exact, clickedText: opt.text, via: 'custom-dropdown', _hint: `selected "${selected}". Verify it matches what you intended before continuing.` };
}
// ---- annotated overlay shot: draw every candidate's rect (numbered), chosen highlighted, click crosshair.
// Injected into the page, CDP-screenshotted, then removed — so the overlay never lingers. Rects are
// viewport-relative (getBoundingClientRect), and we capture the viewport, so fixed-positioned boxes align.
function overlayDrawFn(boxes, chosenIndex, clickX, clickY) {
var id = '__adom_finder_overlay__'; var prev = document.getElementById(id); if (prev) prev.remove();
var o = document.createElement('div'); o.id = id; o.style.cssText = 'position:fixed;inset:0;z-index:2147483646;pointer-events:none;margin:0;padding:0;';
for (var i = 0; i < boxes.length; i++) {
var c = boxes[i], r = c.rect, chosen = (i === chosenIndex);
var col = chosen ? '#00e676' : (c.occluded ? '#ff5252' : '#ffb300');
var b = document.createElement('div');
b.style.cssText = 'position:fixed;left:' + r.x + 'px;top:' + r.y + 'px;width:' + r.w + 'px;height:' + r.h + 'px;border:' + (chosen ? '3px' : '2px') + ' solid ' + col + ';box-sizing:border-box;background:' + (chosen ? 'rgba(0,230,118,0.14)' : 'transparent') + ';';
var lab = document.createElement('div'); lab.textContent = i + (chosen ? ' ★' : '') + (c.occluded ? ' ⛔' : '');
lab.style.cssText = 'position:absolute;top:-15px;left:0;font:bold 12px/14px monospace;color:#000;background:' + col + ';padding:0 4px;border-radius:3px;white-space:nowrap;';
b.appendChild(lab); o.appendChild(b);
}
if (clickX != null && clickY != null) {
var ch = document.createElement('div'); ch.style.cssText = 'position:fixed;left:' + (clickX - 11) + 'px;top:' + (clickY - 11) + 'px;width:22px;height:22px;pointer-events:none;';
ch.innerHTML = '<svg width="22" height="22"><line x1="11" y1="0" x2="11" y2="22" stroke="#2979ff" stroke-width="2"/><line x1="0" y1="11" x2="22" y2="11" stroke="#2979ff" stroke-width="2"/><circle cx="11" cy="11" r="7" fill="none" stroke="#2979ff" stroke-width="2"/></svg>';
o.appendChild(ch);
}
(document.body || document.documentElement).appendChild(o);
}
function overlayClearFn() { var o = document.getElementById('__adom_finder_overlay__'); if (o) o.remove(); }
async function annotateShot(tabId, candidates, chosenIndex, x, y) {
try {
const boxes = candidates.map((c) => ({ rect: c.rect, occluded: !!c.occluded }));
await chrome.scripting.executeScript({ target: { tabId }, func: overlayDrawFn, args: [boxes, chosenIndex == null ? -1 : chosenIndex, x == null ? null : x, y == null ? null : y] });
const shot = await screenshot(tabId, { fullPage: false, maxWidth: 1568, resize: true });
await chrome.scripting.executeScript({ target: { tabId }, func: overlayClearFn }).catch(() => {});
return { base64: shot.base64, width: shot.width, height: shot.height };
} catch (e) {
try { await chrome.scripting.executeScript({ target: { tabId }, func: overlayClearFn }); } catch (_) {}
return { __error: String((e && e.message) || e) };
}
}
// Build the debug `finder` object (incl. shots + annotatedShot as base64; the bridge spills them to files).
async function buildFinder(tabId, query, candidates, chosenIndex, x, y, withShots) {
const finder = { query, chosenIndex, candidates };
if (withShots !== false) {
try { const raw = await screenshot(tabId, { fullPage: false, maxWidth: 1568, resize: true }); if (raw && raw.base64) finder.shots = [{ label: 'viewport-raw', base64: raw.base64 }]; } catch (e) {}
const ann = await annotateShot(tabId, candidates, chosenIndex, x, y);
if (ann && ann.base64) finder.annotatedShot = { base64: ann.base64, width: ann.width, height: ann.height };
else if (ann && ann.__error) finder.annotatedShotError = ann.__error;
}
return finder;
}
// Resolve target xy. When `rich`, also return the full candidate set + chosenIndex + structured query.
async function resolveCoords(tabId, args, rich) {
if (args.selector) {
if (rich) {
const cands = await evaluate(tabId, `(${selectorCandidatesFn.toString()})(${JSON.stringify(args.selector)})`);
const list = Array.isArray(cands) ? cands : [];
const ci = Math.max(0, list.findIndex((c) => c.chosen));
if (!list.length) return { found: false, candidates: [], chosenIndex: -1, query: { selector: args.selector, intent: args.intent } };
const el = list[ci] || list[0];
return { found: true, x: el.x, y: el.y, info: { selector: args.selector, matchedCount: list.length, clickedRect: el.rect, clickedText: el.text }, candidates: list, chosenIndex: ci, query: { selector: args.selector, intent: args.intent } };
}
const r = await evaluate(tabId, scoreExpr(args.selector, !!args.firstMatch));
if (!r || !r.found) return { found: false };
return { found: true, x: r.x, y: r.y, info: { selector: args.selector, matchedCount: r.count, clickedRect: r.rect, clickedText: r.text, modalDetected: r.modal } };
}
if (args.text) { // click/hover BY VISIBLE TEXT — the smart-click path
const f = await find(tabId, args.text, rich ? 16 : 10, rich);
if (!f.ok || !f.elements.length) return { found: false, candidates: [], chosenIndex: -1, query: { text: args.text, role: args.role, intent: args.intent } };
const el = f.elements[0];
const list = rich ? f.elements.map((c, i) => ({ index: i, ...c, chosen: i === 0 })) : null;
return { found: true, x: el.x, y: el.y, info: { matchedText: el.text, matchedTag: el.tag, via: 'text', candidates: f.count }, candidates: list, chosenIndex: rich ? 0 : undefined, query: { text: args.text, role: args.role, intent: args.intent } };
}
return { found: true, x: args.x, y: args.y, info: {}, candidates: rich ? [] : undefined, chosenIndex: rich ? -1 : undefined, query: { intent: args.intent } };
}
// type ∈ move | hover | click | dblclick | rightclick | key | type
// args.cursor:true shows the gliding visual cursor (demos); off by default for fast automation.
export async function inputDispatch(tabId, args) {
await attach(tabId);
const _m = modalGuard(tabId); if (_m && !args.resolveOnly) return _m; // a native dialog is blocking the page
const type = (args.type || 'click').toLowerCase();
const showCursor = args.cursor === true;
const resolveOnly = args.resolveOnly === true; // dry-run: resolve + return finder, dispatch NOTHING
const wantFinder = resolveOnly || args.debug === true; // opt-in; default responses stay lean
try {
if (type === 'key') {
if (resolveOnly) return { ok: true, resolveOnly: true, dispatched: false, key: args.key, _hint: 'key press has no finder target to resolve' };
await pressKey(tabId, args.key); return { ok: true, dispatched: 'key', key: args.key };
}
if (type === 'type') {
let finder = null;
if (wantFinder) {
if (args.selector) { const c = await resolveCoords(tabId, args, true); finder = await buildFinder(tabId, c.query || { selector: args.selector }, c.candidates || [], c.chosenIndex == null ? -1 : c.chosenIndex, c.x, c.y); }
else finder = { query: { intent: args.intent }, chosenIndex: -1, candidates: [], _note: 'typing into the focused field — no selector to resolve' };
}
if (resolveOnly) return { ok: true, resolveOnly: true, dispatched: false, ...(finder ? { finder } : {}) };
if (args.selector) {
const f = await evaluate(tabId, focusExpr(args.selector));
if (!f || !f.found) return { ok: false, error: `selector not found: ${args.selector}`, _hint: 'check the selector or wait for the field', ...(finder ? { finder } : {}) };
}
await send(tabId, 'Input.insertText', { text: String(args.text || '') });
return { ok: true, dispatched: 'type', length: String(args.text || '').length, via: args.selector ? 'selector' : 'focused', ...(finder ? { finder } : {}) };
}
// pointer types: move | hover | click | dblclick | rightclick | wheel
// nb-bridge#41 item 2: an UNKNOWN type used to fall through to a click (four "zoom" attempts became four
// clicks on a canvas editor with a tool armed). Refuse it by name instead.
const KNOWN = ['click', 'dblclick', 'doubleclick', 'rightclick', 'contextmenu', 'move', 'hover', 'wheel', 'mousewheel', 'scroll'];
if (!KNOWN.includes(type)) {
return { ok: false, errorCode: 'unknown_input_type', dispatched: false,
error: `unknown_input_type: "${args.type}" is not an input type I dispatch - refusing rather than guessing (an unknown type used to become a CLICK).`,
_hint: 'type must be one of click | dblclick | rightclick | move | hover | wheel | type | key. For a mouse wheel / canvas zoom use type:"wheel" with deltaX/deltaY (or nbrowser_scroll).' };
}
const isWheel = (type === 'wheel' || type === 'mousewheel' || type === 'scroll');
const c = await resolveCoords(tabId, args, wantFinder);
if (!c.found) {
const out = { ok: false, error: `target not found: ${args.selector || args.text || '(no selector/text)'}`, _hint: 'element not visible/present; wait or refine the selector/text' };
if (wantFinder) out.finder = { query: c.query || {}, chosenIndex: -1, candidates: c.candidates || [] };
return out;
}
const { x, y, info } = c;
// GUARD ambiguous_target — clicking BY TEXT that matches >1 visible element is refused; demand a
// disambiguating selector (or pass acceptAmbiguous:true / index to override). Stops the AI clicking the
// wrong one of two "Next"/"Back" buttons or three "Yes" radios.
if (/click/.test(type) && info.via === 'text' && Number(info.candidates) > 1 && args.acceptAmbiguous !== true && args.index == null && !resolveOnly) {
return { ok: false, errorCode: 'ambiguous_target',
error: `ambiguous_target: text "${args.text}" matches ${info.candidates} visible elements — refusing to guess which to click.`,
_hint: 'Pass a unique selector (nbrowser_click {selector:"…"}), or {debug:true} to see the numbered candidate overlay and pick one, or acceptAmbiguous:true to accept the top-scored match.',
...(wantFinder ? { finder: await buildFinder(tabId, c.query || {}, c.candidates || [], c.chosenIndex == null ? 0 : c.chosenIndex, x, y) } : {}) };
}
// GUARD stale_coordinates — a raw {x,y} pointer verb is only safe if the page hasn't scrolled since
// those coords were computed. Require the scroll baseline (atScrollX/atScrollY, from a screenshot's
// coordMap) and refuse if it drifted — a cached pixel silently becomes a DIFFERENT element after a
// scroll/re-render (the "Next" that turned into "Back"). Re-resolve by selector/text instead.
// nb-bridge#41 item 1: the gate applies to CLICKS only. A hover, move or wheel is read-only pointer motion:
// a stale hover shows the wrong tooltip, it never presses the wrong control. Those accept raw {x,y}.
if (/click|contextmenu/.test(type) && !args.selector && !args.text && (args.x != null || args.y != null) && !resolveOnly) {
const live = await evaluate(tabId, '({sx:Math.round(scrollX),sy:Math.round(scrollY)})');
const bx = args.atScrollX, by = args.atScrollY;
if (bx == null && by == null) {
return { ok: false, errorCode: 'stale_coordinates',
error: `stale_coordinates: a raw {x,y} ${type} carries no freshness proof - the page may have scrolled/re-rendered since those coords were valid.`,
_hint: 'Prefer nbrowser_click {selector:"…"} or {text:"…"} (re-resolved live). If you must click by pixel, pass atScrollX/atScrollY from your latest screenshot coordMap so I can verify the page has not scrolled.' };
}
if ((bx != null && Math.abs(Number(bx) - live.sx) > 2) || (by != null && Math.abs(Number(by) - live.sy) > 2)) {
return { ok: false, errorCode: 'stale_coordinates',
error: `stale_coordinates: the page scrolled since these coords were captured (baseline ${bx},${by}; now ${live.sx},${live.sy}) — {x:${args.x},y:${args.y}} now points at different content.`,
_hint: 'Re-screenshot and re-resolve, or click by selector/text. Never reuse pixel coordinates across a scroll or re-render.' };
}
}
// Build the debug finder (candidates + scores + annotated overlay shot) BEFORE acting, so a
// resolveOnly dry-run and a debug click see the same pre-action page state.
const finder = wantFinder ? await buildFinder(tabId, c.query || {}, c.candidates || [], c.chosenIndex == null ? 0 : c.chosenIndex, x, y) : null;
const fin = finder ? { finder } : {};
if (resolveOnly) return { ok: true, resolveOnly: true, dispatched: false, x, y, ...info, ...fin };
// The cursor label narrates intention (Hydrogen-style). args.intent overrides; else auto from element text.
const intent = args.intent || args.label;
const tgt = (info.clickedText || info.matchedText || '').trim().slice(0, 40);
if (showCursor) { await cursor(tabId, 'move', x, y, intent || phaseLabel(type, tgt, 'find')); await sleep(args.cursorGlideMs || 450); }
if (isWheel) {
// Real CDP wheel (isTrusted): what a canvas camera (Babylon ArcRotateCamera, three.js OrbitControls) and
// a scrollable pane both listen for. deltaY < 0 zooms in / scrolls up, > 0 zooms out / scrolls down.
const deltaX = Number(args.deltaX) || 0, deltaY = Number(args.deltaY) || 0;
if (!deltaX && !deltaY) return { ok: false, errorCode: 'bad_args', error: 'wheel needs deltaX and/or deltaY (px; deltaY:-300 zooms in / scrolls up)' };
const steps = Math.max(1, Math.min(20, Number(args.steps) || 1));
const before = await evaluate(tabId, '({sx:Math.round(scrollX),sy:Math.round(scrollY)})');
await send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y });
for (let i = 0; i < steps; i++) {
await send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseWheel', x, y, deltaX: deltaX / steps, deltaY: deltaY / steps });
if (steps > 1) await sleep(Number(args.stepDelayMs) || 40);
}
await sleep(Number(args.settleMs) || 150);
const after = await evaluate(tabId, '({sx:Math.round(scrollX),sy:Math.round(scrollY)})');
return { ok: true, dispatched: 'wheel', x, y, deltaX, deltaY, steps, scrollBefore: before, scrollAfter: after,
pageScrolled: !!(before && after && (before.sx !== after.sx || before.sy !== after.sy)), ...info, ...fin,
_hint: 'isTrusted wheel at that point. pageScrolled:false is normal on a canvas (the app consumed the wheel, e.g. a camera zoom); verify the effect with nbrowser_screenshot or an eval of the app state.' };
}
if (type === 'move' || type === 'hover') {
if (showCursor) await cursor(tabId, 'move', x, y, intent || phaseLabel(type, tgt, 'act'));
// Approach from just outside, then settle on center, then dwell — a single mouseMoved
// often fails to fire mouseover/mouseenter or let a CSS-transition tooltip paint. Moving
// INTO the element generates the enter/over events real hover needs, and the dwell lets
// delayed/animated tooltips appear before the caller screenshots.
await send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: x - 24, y: y - 24 });
await sleep(40);
await send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }); // triggers :hover / mouseover / tooltips
await sleep(args.hoverDwellMs || 600);
return { ok: true, dispatched: type, x, y, intentLabel: intent || phaseLabel(type, tgt, 'act'), ...info, ...fin };
}
if (showCursor) await cursor(tabId, 'click', x, y, intent || phaseLabel(type, tgt, 'act')); // ripple + label
if (type === 'dblclick' || type === 'doubleclick') {
await mouseClick(tabId, x, y, 'left', 1); await mouseClick(tabId, x, y, 'left', 2);
return { ok: true, dispatched: 'dblclick', x, y, ...info, ...fin };
}
if (type === 'rightclick' || type === 'contextmenu') {
await mouseClick(tabId, x, y, 'right', 1);
return { ok: true, dispatched: 'rightclick', x, y, ...info, ...fin };
}
await mouseClick(tabId, x, y, args.button || 'left', args.clickCount || 1);
return { ok: true, dispatched: 'click', via: args.selector ? 'selector' : 'coords', x, y, ...info, ...fin };
} finally {
bumpIdle(tabId);
}
}
// ---- nbrowser_autologin: inject the bridge-decrypted saved password into the password field ----------
// The bridge decrypts the saved password on the laptop and passes it as args.__secret (laptop-local, over
// native-messaging stdio). This handler injects it via CDP isTrusted keystrokes and submits. SECURITY:
// __secret is NEVER returned, logged, or echoed — only {ok, loggedIn, needs2FA, host, url}.
function probeLoginFn() {
function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01; }
function info(el) { if (!el) return null; var r = el.getBoundingClientRect(); return { x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), name: el.name || '', id: el.id || '', hasVal: !!(el.value && el.value.length) }; }
var pw = [].slice.call(document.querySelectorAll('input[type=password]')).filter(vis)[0];
var user = [].slice.call(document.querySelectorAll('input[type=email],input[autocomplete=username],input[name*="user" i],input[name*="email" i],input[id*="user" i],input[id*="email" i],input[type=text]')).filter(vis)[0];
return { pw: info(pw), user: info(user), url: location.href };
}
function assessLoginFn() {
var url = location.href; var body = ((document.body && document.body.innerText) || '').toLowerCase();
var hasPw = !!document.querySelector('input[type=password]');
var twofa = /(verification code|two[- ]factor|2[- ]step|enter the code|authenticator app|one[- ]time (code|passcode)|security code|6[- ]digit|passcode we (sent|texted)|verify it's you)/.test(body);
var failed = /(incorrect|wrong password|invalid (email|password)|try again|password isn't right)/.test(body) && hasPw;
return { url: url, hasPw: hasPw, twofa: twofa, failed: failed };
}
async function focusType(tabId, fld, text) {
await mouseClick(tabId, fld.x, fld.y); await sleep(70);
if (fld.hasVal) { for (let i = 0; i < 60; i++) await pressKey(tabId, 'Backspace'); } // clear pre-fills
await send(tabId, 'Input.insertText', { text: String(text) });
}
export async function autologin(tabId, args) {
await attach(tabId);
const secret = args && args.__secret; // injected by the bridge; do NOT log/return this
const username = args && args.username;
try {
if (!secret) return { ok: false, error: 'no credential available' + (args && args.__secretError ? ': ' + args.__secretError : ''), _hint: 'autologin needs confirm:true + host, and a saved login for that host in this profile. Decryption happens in the in-Chrome native host; if __secretError shows, the saved password could not be decrypted/found.' };
let probe = await evaluate(tabId, `(${probeLoginFn.toString()})()`);
// Split form: no password field yet → fill username + advance to the password step.
if (probe && !probe.pw) {
if (probe.user && username) {
await focusType(tabId, probe.user, String(username));
await pressKey(tabId, 'Enter');
await sleep(1800);
probe = await evaluate(tabId, `(${probeLoginFn.toString()})()`);
}
if (!probe || !probe.pw) return { ok: false, loggedIn: false, step: 'username', url: (probe && probe.url) || '', _hint: 'Filled username + advanced but no password field appeared yet — wait for the password step (or click Sign in/Continue), then call nbrowser_autologin again.' };
}
// Username present + empty on the same page → fill it (not secret).
if (probe.user && !probe.user.hasVal && username) await focusType(tabId, probe.user, String(username));
// Inject the secret into the password field via real keystrokes (fires framework onChange).
await focusType(tabId, probe.pw, secret);
await sleep(120);
await pressKey(tabId, 'Enter'); // submit
await sleep(2800);
const a = await evaluate(tabId, `(${assessLoginFn.toString()})()`);
const url = (a && a.url) || '';
const needs2FA = !!(a && a.twofa);
const loggedIn = !needs2FA && a && !a.hasPw && !a.failed;
return { ok: true, loggedIn, needs2FA, host: args.host, url,
_hint: loggedIn ? 'Logged in (no password form remains).' : needs2FA ? '2FA gate detected — a human verification code (or a TOTP/passkey path) is required; nbrowser_screenshot to see it.' : a && a.failed ? 'Credentials rejected — saved password may be stale.' : 'Submitted but a password form is still present; the submit button may differ — nbrowser_screenshot, then nbrowser_click the explicit Sign in button.' };
} finally { bumpIdle(tabId); }
}
// ---- nbrowser_login_form: detect a login form's state + recommend the SMARTEST path -----------------
// The #1 rule of login automation: if the browser already AUTOFILLED the form, just SUBMIT it (fully
// background, no autofill popup, no SSO). This inspects the current page and tells the AI which path to take.
function loginFormFn() {
function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01; }
function txt(b) { return ((b.innerText || b.value || b.getAttribute('aria-label') || '') + '').replace(/\s+/g, ' ').trim(); }
function sel(el) { if (!el) return null; if (el.id) return '#' + el.id; if (el.name) return (el.tagName.toLowerCase() + '[name="' + el.name + '"]'); return el.tagName.toLowerCase(); }
var pw = [].slice.call(document.querySelectorAll('input[type=password]')).filter(vis)[0];
var user = [].slice.call(document.querySelectorAll('input[type=email],input[autocomplete=username],input[name*="user" i],input[name*="email" i],input[id*="email" i],input[id*="user" i],input[type=text]')).filter(vis)[0];
var btns = [].slice.call(document.querySelectorAll('button,input[type=submit],[role=button],a[href]')).filter(vis);
var submit = btns.find(function (b) { return /^(sign in|log ?in|continue|submit|next|verify)$/i.test(txt(b)); })
|| btns.find(function (b) { return /sign in|log ?in|continue|submit/i.test(txt(b)) && !/google|microsoft|apple|sso|with/i.test(txt(b)); })
|| (pw && pw.form && pw.form.querySelector('button[type=submit],input[type=submit],button'));
var sso = btns.filter(function (b) { return /(sign in|continue|log ?in) with (google|microsoft|apple|github|okta|saml|sso)|use (sso|single sign)/i.test(txt(b)); }).map(txt).slice(0, 5);
var twofa = /(verification code|two[- ]factor|2[- ]step|authenticator app|enter the (\d|six)[- ]?digit|security code|one[- ]time)/i.test((document.body && document.body.innerText || '').toLowerCase());
return {
hasPasswordField: !!pw,
passwordFilled: !!(pw && pw.value && pw.value.length > 0),
usernameField: !!user,
usernameFilled: !!(user && user.value && user.value.length > 0),
usernameValue: user ? (user.value || '').slice(0, 60) : '', // email/username is NOT a secret — lets the AI confirm the right account
submitText: submit ? txt(submit) : '',
submitSelector: sel(submit),
ssoOptions: sso,
twoFactorPrompt: twofa,
url: location.href,
};
}
export async function detectLoginForm(tabId) {
await attach(tabId);
const f = await evaluate(tabId, `(${loginFormFn.toString()})()`);
bumpIdle(tabId);
if (!f || f.__error) return { ok: false, error: (f && f.__error) || 'eval failed' };
let recommendation, action;
if (f.twoFactorPrompt) { recommendation = '2FA gate — needs a human verification code (authenticator/SMS). Surface it to the user; cannot be automated.'; action = '2fa'; }
else if (f.hasPasswordField && f.passwordFilled) { recommendation = `✅ FORM IS AUTOFILLED${f.usernameValue ? ' for ' + f.usernameValue : ''} — the browser pre-filled the credentials. JUST SUBMIT IT (fully background, no popup/SSO): nbrowser_click {text:"${f.submitText || 'Sign in'}"}${f.submitSelector ? ' or {selector:"' + f.submitSelector + '"}' : ''}.`; action = 'submit'; }
else if (f.ssoOptions.length) { recommendation = `Form not autofilled, but SSO is available (${f.ssoOptions.join(', ')}). Prefer SSO — clicking it uses the existing identity-provider session (background, no password): nbrowser_click {text:"${f.ssoOptions[0]}"}.`; action = 'sso'; }
else if (f.hasPasswordField) { recommendation = 'Password form present but NOT autofilled. Fill the email (background) if split, then summon the browser autofill popup (~1s foreground: click the password field → desktop_screenshot_window owned popup → click the saved-login row). See the driving-the-desktop skill.'; action = 'autofill-popup'; }
else if (f.usernameField) { recommendation = 'Email/username step (no password field yet). Type the email (background) + click Continue, then re-check nbrowser_login_form on the password step.'; action = 'username-step'; }
else { recommendation = 'No login form detected on this page.'; action = 'none'; }
return { ok: true, action, recommendation, ...f, _hint: 'ALWAYS call this before driving a login. If passwordFilled=true, the browser already autofilled it — just submit (background); never reach for the autofill popup or SSO when the form is already filled.' };
}
// ---- native JS dialogs (alert/confirm/prompt/beforeunload) — modal detection ----------------------
// These are NOT OS child windows (no hwnd) - Chrome renders them itself and they BLOCK the page's main
// thread, so an AI that doesn't know one is up will keep firing clicks/eval into a frozen page (real
// incident: an Intuit "questionnaire saved" alert sat over the form while the AI kept trying to fill it).
// We catch Page.javascriptDialogOpening (the debugger is attached during any CDP drive), record it, and
// surface it on the session's NEXT verb (modal_open) with an easy way to dismiss it.
const _openDialogs = new Map(); // tabId -> { type, message, defaultPrompt, ts } (currently OPEN)
const _dialogPolicy = new Map(); // tabId -> 'auto' (default: never wedge) | 'manual' (caller answers)
const _recentDialogs = []; // ring for nbrowser_events - DRAINED by whoever polls events
// ⚠️ _recentDialogs is drained by ANY events consumer (another AI thread polling nbrowser_events on this
// profile empties it), so it cannot double as the "was a dialog just auto-handled here?" memory - the
// alreadyHandled recovery and the click dialogOpened check lost their evidence to a foreign drain
// (found 2026-08-22: the Fusion thread's event polling raced the verb battery). Per-tab last-dialog
// memory lives here instead, and nothing drains it.
const _lastDialog = new Map(); // tabId -> { dialogType, message, autoHandled, accepted, ts }
export function dialogFor(tabId) { return _openDialogs.get(Number(tabId)) || null; }
export function lastDialogFor(tabId) { return _lastDialog.get(Number(tabId)) || null; }
export function drainDialogs() { const d = _recentDialogs.splice(0); return d; }
// nbrowser_handle_dialog — control native JS dialogs. GROUND TRUTH (measured 2026-08-22, Chrome 139,
// john's laptop): while a chrome.debugger client is attached, CHROME ITSELF cancels page dialogs -
// Page.javascriptDialogOpening never reaches the extension (confirm() returns false instantly, our
// listener sees nothing). So the CDP arming design ("dialog stays open, answer it later") CANNOT work,
// and it silently didn't: policy:'manual' armed a listener Chrome never called. The honest, working
// design is a PAGE-LEVEL PRE-ANSWER: arming overrides window.confirm/alert/prompt in the page to record
// the call and return YOUR preset answer, and disarming restores the originals and reports what was
// seen. The CDP path below stays for the (non-attached / other-browser) case where it does fire.
export async function handleDialog(tabId, accept, promptText, policy) {
await attach(tabId);
const id = Number(tabId);
if (policy === 'manual' || policy === 'auto') {
_dialogPolicy.set(id, policy);
if (policy === 'manual') {
const armed = await evaluate(tabId,
`(function(){ if (window.__nbDlg && window.__nbDlg.armed) { window.__nbDlg.accept=${accept !== false}; window.__nbDlg.text=${JSON.stringify(promptText != null ? String(promptText) : null)}; return 're-armed'; }
var w=window; w.__nbDlg={armed:true, accept:${accept !== false}, text:${JSON.stringify(promptText != null ? String(promptText) : null)}, seen:[], orig:{c:w.confirm,a:w.alert,p:w.prompt}};
w.confirm=function(m){ w.__nbDlg.seen.push({type:'confirm',message:String(m),answered:w.__nbDlg.accept,ts:Date.now()}); return w.__nbDlg.accept; };
w.alert=function(m){ w.__nbDlg.seen.push({type:'alert',message:String(m),answered:true,ts:Date.now()}); };
w.prompt=function(m,d){ var ans=w.__nbDlg.accept ? (w.__nbDlg.text!=null?w.__nbDlg.text:(d||'')) : null; w.__nbDlg.seen.push({type:'prompt',message:String(m),answered:ans,ts:Date.now()}); return ans; };
return 'armed'; })()`);
return { ok: true, policy: 'manual', armed: true, mode: String(armed), preAnswer: { accept: accept !== false, text: promptText != null ? String(promptText) : null }, tabId: String(id),
_hint: `Manual mode ARMED as a pre-answer: the NEXT dialog(s) on this tab are answered ${accept !== false ? 'ACCEPT (OK/Yes)' : 'DISMISS (Cancel)'} the instant they fire${promptText != null ? `, prompts answered "${String(promptText).slice(0, 40)}"` : ''}. This is a page-level override (Chrome cancels dialogs itself while a debugger is attached, so a dialog can never be held open for a later answer). Do the action now, then policy:"auto" to disarm and get the list of dialogs seen. The override does NOT survive a page navigation - re-arm after navigating.` };
}
// policy:'auto' - restore the page's own dialog functions and report what the arming saw.
const seen = await evaluate(tabId,
`(function(){ var w=window, d=w.__nbDlg; if (!d) return []; if (d.orig){ w.confirm=d.orig.c; w.alert=d.orig.a; w.prompt=d.orig.p; } var s=d.seen||[]; delete w.__nbDlg; return s; })()`).catch(() => []);
return { ok: true, policy: 'auto', disarmed: true, seen: Array.isArray(seen) ? seen : [], tabId: String(id),
_hint: 'Auto mode restored (page dialog functions un-overridden). While driving, Chrome auto-cancels dialogs the moment they open (confirm/prompt return the dismiss value, alerts just close) so the page can never wedge - to ANSWER one deliberately, arm the pre-answer first: {policy:"manual", accept:true|false, text?}. `seen` lists every dialog the arming answered.' };
}
const dlg = _openDialogs.get(id);
try { await send(tabId, 'Page.handleJavaScriptDialog', { accept: accept !== false, promptText: promptText != null ? String(promptText) : undefined }); }
catch (e) {
const auto = ((_lastDialog.get(id) || {}).autoHandled ? _lastDialog.get(id) : null)
|| [..._recentDialogs].reverse().find((x) => x.tabId === String(id) && x.autoHandled);
if (auto) return { ok: true, handled: false, alreadyHandled: true, was: auto, tabId: String(id),
_hint: `No dialog was open: a ${auto.dialogType} ("${String(auto.message).slice(0, 80)}") was AUTO-${auto.accepted ? 'ACCEPTED' : 'DISMISSED'} the moment it opened, so the page never blocked. To answer one yourself, arm the PRE-ANSWER first: nbrowser_handle_dialog {sessionId, policy:"manual", accept:true}, then do the action, then policy:"auto" to disarm.` };
// Pre-answer memory: the page-level arming records every dialog it answered in window.__nbDlg.seen.
const seen = await evaluate(tabId, '(window.__nbDlg && window.__nbDlg.seen) || []').catch(() => []);
if (Array.isArray(seen) && seen.length) {
const last = seen[seen.length - 1];
return { ok: true, handled: false, alreadyHandled: true, was: last, tabId: String(id),
_hint: `No dialog is open: your ARMED pre-answer already answered a ${last.type} ("${String(last.message).slice(0, 80)}") with ${JSON.stringify(last.answered)}. Disarm + collect the full list with {policy:"auto"}.` };
}
return { ok: false, error: 'no dialog to handle (it may have already closed): ' + ((e && e.message) || e),
_hint: 'No dialog is open. NOTE: while nbe is driving, Chrome cancels page dialogs the moment they fire (confirm returns false, alerts close) - the page can never wedge, but a dialog also can never be caught open. To ANSWER the next dialog deliberately, arm a pre-answer BEFORE the action: nbrowser_handle_dialog {sessionId, policy:"manual", accept:true|false, text?}, do the action, then {policy:"auto"} to disarm and see what was answered.' };
}
_openDialogs.delete(Number(tabId));
return { ok: true, handled: true, accepted: accept !== false, was: dlg || null, tabId: String(tabId), _hint: 'Dialog dismissed — the page is unblocked. Continue driving; re-screenshot to see the current state.' };
}
// Guard called at the start of page-ACTION verbs: if a native dialog is blocking this tab, refuse the
// action and tell the AI to deal with the modal first (it cannot succeed while the page is frozen).
export function modalGuard(tabId) {
const dlg = _openDialogs.get(Number(tabId));
if (!dlg) return null;
return { ok: false, errorCode: 'modal_open',
error: `modal_open: a native ${dlg.type} dialog is blocking this tab: "${(dlg.message || '').slice(0, 140)}". The page is FROZEN until it is handled - your clicks/typing/eval will not work.`,
dialog: { type: dlg.type, message: dlg.message },
_hint: `Handle it first: nbrowser_handle_dialog {sessionId, accept:true} to press OK/Yes/Leave, or accept:false to Cancel/Stay${dlg.type === 'prompt' ? ', with text:"…" to answer a prompt' : ''}. Then re-screenshot and continue.` };
}
// ---- nbrowser_errors: buffered console/page errors per tab ------------------
const errorBuffers = new Map(); // tabId -> [{type,text,ts}]
chrome.debugger.onEvent.addListener((source, method, params) => {
const tabId = source.tabId;
if (tabId == null) return;
if (method === 'Page.javascriptDialogOpening') {
const d = { type: params.type || 'alert', message: params.message || '', defaultPrompt: params.defaultPrompt || '', ts: Date.now() };
_openDialogs.set(tabId, d);
// AUTO-HANDLE (nb-bridge #21). A native dialog BLOCKS the page's main thread, so if nobody answers
// it every later eval/click/screenshot on that tab hangs until the window is closed - a driven
// click on any confirm() used to kill the whole drive. The docs always promised auto-dismissal
// while driving; this is it. Policy per dialog TYPE, biased to the SAFE answer:
// alert -> accept (only has OK) confirm -> DISMISS (do not perform the guarded action)
// prompt -> dismiss (never invent input) beforeunload -> accept (we asked to navigate)
// A caller that genuinely wants OK sets policy:'manual' (nbrowser_handle_dialog) and answers itself.
const policy = _dialogPolicy.get(tabId) || 'auto';
const accept = d.type === 'alert' || d.type === 'beforeunload';
if (policy === 'auto') {
send(tabId, 'Page.handleJavaScriptDialog', { accept }).then(() => {
_openDialogs.delete(tabId);
_lastDialog.set(tabId, { dialogType: d.type, message: d.message, autoHandled: true, accepted: accept, ts: d.ts });
_recentDialogs.push({ event: 'dialog', tabId: String(tabId), dialogType: d.type, message: d.message, autoHandled: true, accepted: accept, ts: d.ts,
note: `auto-${accept ? 'accepted' : 'dismissed'} so the page could not wedge; set policy:"manual" via nbrowser_handle_dialog BEFORE the action if you need to answer it yourself` });
while (_recentDialogs.length > 20) _recentDialogs.shift();
}).catch(() => {
_recentDialogs.push({ event: 'dialog', tabId: String(tabId), dialogType: d.type, message: d.message, autoHandled: false, ts: d.ts });
while (_recentDialogs.length > 20) _recentDialogs.shift();
});
return;
}
_recentDialogs.push({ event: 'dialog', tabId: String(tabId), dialogType: d.type, message: d.message, autoHandled: false, ts: d.ts });
while (_recentDialogs.length > 20) _recentDialogs.shift();
return;
}
if (method === 'Page.javascriptDialogClosed') { const o = _openDialogs.get(tabId); _openDialogs.delete(tabId); if (o) _recentDialogs.push({ event: 'dialog_closed', tabId: String(tabId), dialogType: o.type, result: params.result, ts: Date.now() }); return; }
if (method === 'Page.fileChooserOpened') {
// armed by nbrowser_set_file_input {armForNextChooser:true}: satisfy the page's chooser with the
// staged files - silently, in the background, no OS dialog ever appears.
const arm = _chooserArm.get(tabId);
if (arm && params.backendNodeId) {
send(tabId, 'DOM.setFileInputFiles', { files: arm.files, backendNodeId: params.backendNodeId })
.then(() => { _chooserServed.set(tabId, { files: arm.files.map((f) => String(f).split(/[\\/]/).pop()), ts: Date.now() }); })
.catch(() => {})
.finally(() => { if (arm.timer) clearTimeout(arm.timer); _chooserArm.delete(tabId); send(tabId, 'Page.setInterceptFileChooserDialog', { enabled: false }).catch(() => {}); });
}
return;
}
let entry = null;
if (method === 'Runtime.exceptionThrown') {
const e = params.exceptionDetails || {};
entry = { type: 'exception', text: (e.exception && (e.exception.description || e.exception.value)) || e.text, ts: Date.now() };
} else if (method === 'Runtime.consoleAPICalled' && params.type === 'error') {
entry = { type: 'console.error', text: (params.args || []).map((a) => a.value ?? a.description ?? '').join(' '), ts: Date.now() };
} else if (method === 'Log.entryAdded' && params.entry && params.entry.level === 'error') {
entry = { type: 'log', text: params.entry.text, ts: Date.now() };
}
if (!entry) return;
const buf = errorBuffers.get(tabId) || [];
buf.push(entry);
errorBuffers.set(tabId, buf.slice(-200));
});
export async function getErrors(tabId, clear = true) {
const buf = errorBuffers.get(tabId) || [];
if (clear) errorBuffers.delete(tabId);
return { ok: true, errors: buf, count: buf.length, tabId: String(tabId) };
}
// ---- PNG helpers (OffscreenCanvas, runs in the service worker) -------------
async function bitmapFromB64(b64) {
const bytes = Uint8Array.from(atob(b64), (c) => c.charCodeAt(0));
return createImageBitmap(new Blob([bytes], { type: 'image/png' }));
}
async function pngDims(b64) {
const bmp = await bitmapFromB64(b64);
const out = { base64: b64, width: bmp.width, height: bmp.height };
bmp.close && bmp.close();
return out;
}
async function resizePng(b64, maxWidth) {
const bmp = await bitmapFromB64(b64);
const { width, height } = bmp;
if (width <= maxWidth) { bmp.close && bmp.close(); return { base64: b64, width, height, resized: false }; }
const w = maxWidth;
const h = Math.round(height * (maxWidth / width));
const canvas = new OffscreenCanvas(w, h);
canvas.getContext('2d').drawImage(bmp, 0, 0, w, h);
bmp.close && bmp.close();
const blob = await canvas.convertToBlob({ type: 'image/png' });
const buf = new Uint8Array(await blob.arrayBuffer());
let bin = '';
for (let i = 0; i < buf.length; i++) bin += String.fromCharCode(buf[i]);
return { base64: btoa(bin), width: w, height: h, origWidth: width, origHeight: height, resized: true };
}
// ── #5 Robust click: click, then report whether the page ACTUALLY changed ──────
// Captures a light page signature (url, readyState, visible-text length) before and after the click,
// so callers can detect "click dispatched but nothing happened" (disabled/covered element, missed
// target) and retry. Also scrolls the target into view (resolveCoords already centers it).
export async function pageSig(tabId) {
// t = a cheap rolling hash of the visible text, so a SMALL local mutation (a button relabelling
// itself from "Click Me" to "Clicked") is detected. Length alone missed those and reported a
// landed click as changed:false (nb-bridge #20).
try { const r = await evaluate(tabId, "(function(){var s=document.body?document.body.innerText:'';var h=0;for(var i=0;i<s.length;i++){h=(h*31+s.charCodeAt(i))|0;}return {u:location.href, r:document.readyState, n:s.length, t:h};})()"); return r || {}; } catch { return {}; }
}
export async function clickChanged(tabId, args) {
const before = await pageSig(tabId);
let r = await inputDispatch(tabId, { ...args, type: 'click' });
if (r && r.ok === false) return r;
await sleep(Number(args.settleMs) || 500);
let after = await pageSig(tabId);
const didChange = (b, a) => (b.u !== a.u) || (a.t !== undefined && b.t !== undefined && a.t !== b.t) || Math.abs((a.n || 0) - (b.n || 0)) > 3 || (b.r !== a.r && a.r !== 'complete');
let changed = didChange(before, after), retried = false;
// SELF-HEAL (nb-bridge #20). A click-by-text that resolves fine and then lands nowhere is the
// single most confusing failure this bridge had: ok:true, matchedText set, changed:false, DOM
// untouched - because the coordinates were resolved a beat before the page settled (late layout,
// font swap, lazy image) so the point no longer hit the element. Re-resolving and dispatching once
// more costs ~600ms only in the failure case and turns a silent miss into a hit.
// A click that OPENED A DIALOG froze the page - the DOM cannot change, so changed:false is expected
// and the dialog IS the effect. Retrying here queued a second click into the frozen page, which fired
// a SECOND confirm() the moment the first auto-dismissed (found by the verb battery, 2026-08-22).
const dlgNow = _openDialogs.get(Number(tabId));
const _ld = _lastDialog.get(Number(tabId));
const dlgRecent = !changed && ((_ld && (Date.now() - _ld.ts) < 3000) ? _ld
: [..._recentDialogs].reverse().find((x) => x.tabId === String(tabId) && (Date.now() - x.ts) < 3000));
if (!changed && (dlgNow || dlgRecent)) {
const d = dlgNow || dlgRecent;
return { ...r, changed: true, retried: false, dialogOpened: { type: d.type || d.dialogType, message: d.message },
urlBefore: before.u, urlAfter: after.u,
_hint: `The click's effect was a native ${d.type || d.dialogType} dialog ("${String(d.message || '').slice(0, 80)}")${dlgNow ? ' which is OPEN and blocking the page - handle it: nbrowser_handle_dialog {sessionId, accept:true|false}' : ' which was auto-handled (see nbrowser_events)'} - the page DOM did not change and that is expected.` };
}
if (!changed && args.retry !== false) {
await sleep(350);
const r2 = await inputDispatch(tabId, { ...args, type: 'click' });
if (!(r2 && r2.ok === false)) {
await sleep(Number(args.settleMs) || 500);
const after2 = await pageSig(tabId);
if (didChange(before, after2)) { r = r2; after = after2; changed = true; }
retried = true;
}
}
return { ...r, changed, retried, urlBefore: before.u, urlAfter: after.u,
_hint: changed
? (retried ? 'Landed on the SECOND dispatch - the first missed because the page was still settling when the coordinates were resolved. If you see this often on one site, prefer a selector over click-by-text there.' : undefined)
: 'The click DISPATCHED TWICE and the page still did not visibly change (same URL + DOM). The element is probably disabled, covered, or the handler does nothing visible. Re-resolve the selector, nbrowser_wait_for a follow-up element, or verify with nbrowser_eval.' };
}
// ── #7 set_date: fill a react-aria segmented date field (role=spinbutton MM/DD/YYYY) ──
// Focus the field, then type each digit as a real keydown (react-aria segments advance on digit
// keydown - Input.insertText does NOT trigger them). US MM/DD/YYYY order. date = "YYYY-MM-DD".
export async function setDate(tabId, args) {
await attach(tabId);
const selector = args.selector, date = args.date;
if (!selector || !date) return { ok: false, error: 'set_date needs {selector, date:"YYYY-MM-DD"}' };
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(String(date));
if (!m) return { ok: false, error: 'date must be "YYYY-MM-DD"' };
const [, Y, M, D] = m;
const c = await resolveCoords(tabId, { selector }, false);
if (!c.found) return { ok: false, error: `date field not found: ${selector}`, _hint: 'check the selector or wait for the field' };
await mouseClick(tabId, c.x, c.y, 'left', 1); // focus the first segment
await sleep(90);
const order = (String(args.order || 'MDY').toUpperCase() === 'DMY') ? [D, M, Y] : [M, D, Y];
for (const seg of order) { for (const ch of seg) { await pressKey(tabId, ch); await sleep(35); } await sleep(40); }
const val = await evaluate(tabId, `(function(){var e=document.querySelector(${JSON.stringify(selector)});if(!e)return null;return e.value||e.getAttribute('aria-valuetext')||(e.textContent||'').trim().slice(0,40);})()`);
return { ok: true, dispatched: 'set_date', date, order: (String(args.order || 'MDY').toUpperCase() === 'DMY') ? 'DMY' : 'MDY', fieldValue: val, _hint: 'Typed each digit as a real keydown so react-aria segments advance. If fieldValue looks wrong, the field may use DMY order (pass order:"DMY") or a custom widget - screenshot to check.' };
}
// ── #3 TOTP: type the current 6-digit code (computed by the BRIDGE, injected as __code) into the
// focused 2FA field. The agent never sees the code; we only report {typed}. ──
export async function typeTotp(tabId, args) {
const code = args && args.__code;
if (!code) return { ok: false, error: 'no code (register a seed first: nbrowser_register_totp {host, seed})' };
await attach(tabId);
if (args.selector) { const f = await evaluate(tabId, focusExpr(args.selector)); if (!f || !f.found) return { ok: false, error: `2FA field not found: ${args.selector}` }; }
await send(tabId, 'Input.insertText', { text: String(code) });
if (args.submit === true) await pressKey(tabId, 'Enter');
return { ok: true, typed: true, digits: String(code).length, submitted: args.submit === true };
}
// ---- nbrowser_set_file_input: populate <input type=file> over CDP, NO OS picker ------------------
// The OS file dialog cannot be driven with the debugger attached (and it FOREGROUNDS), so uploads go
// through DOM.setFileInputFiles - the same primitive Puppeteer's uploadFile / Playwright's
// setInputFiles use. Two modes:
// direct (default): resolve the input (selector/text -> walk Browse-button/label to its input) and
// inject the files, then dispatch input+change so React/Angular listeners fire.
// armForNextChooser: for UIs that CREATE the input inside the Browse click handler - we intercept
// the page's next file chooser (Page.setInterceptFileChooserDialog) and satisfy it with the staged
// files when Page.fileChooserOpened fires. Auto-disarms after 30s.
// LIMITATION (documented in the bridge hint): the File System Access API (showOpenFilePicker) cannot
// be satisfied over CDP at all - those sites need their drag-drop path.
function findFileInputFn(sel, txt) {
return `(function(){
var sel=${JSON.stringify(sel || null)}, txt=${JSON.stringify(txt || null)};
function fromEl(el){
if(!el) return null;
if(el.matches && el.matches('input[type=file]')) return el;
if(el.tagName==='LABEL' && el.htmlFor){ var t=document.getElementById(el.htmlFor); if(t && t.type==='file') return t; }
var lbl=el.closest && el.closest('label'); if(lbl){ var i=lbl.querySelector('input[type=file]'); if(i) return i; }
if(el.querySelector){ var i2=el.querySelector('input[type=file]'); if(i2) return i2; }
var form=el.closest && el.closest('form'); if(form){ var i3=form.querySelector('input[type=file]'); if(i3) return i3; }
var p=el.parentElement; for(var d=0; p && d<4; d++,p=p.parentElement){ var i4=p.querySelector('input[type=file]'); if(i4) return i4; }
return null;
}
var el=null;
if(sel) el=document.querySelector(sel);
else if(txt){ var t=txt.trim().toLowerCase();
var nodes=Array.prototype.slice.call(document.querySelectorAll('button,a,label,input,[role=button],div,span'));
el=nodes.find(function(n){ var s=(n.innerText||n.value||n.getAttribute('aria-label')||'').trim().toLowerCase(); return s && s.indexOf(t)>=0 && s.length < t.length+60; })||null;
}
var inp=fromEl(el);
if(!inp && !sel && !txt) inp=document.querySelector('input[type=file]');
if(!inp){ var all=document.querySelectorAll('input[type=file]'); if(all.length===1) inp=all[0]; }
return inp||null;
})()`;
}
const _chooserArm = new Map(); // tabId -> { files, timer }
const _chooserServed = new Map(); // tabId -> { files, ts }
export async function setFileInput(tabId, args) {
await attach(tabId);
const _m = modalGuard(tabId); if (_m) return _m;
const files = args.filePaths || [];
if (args.chooserStatus === true) {
const s = _chooserServed.get(Number(tabId)) || null;
return { ok: true, armed: _chooserArm.has(Number(tabId)), served: !!s, ...(s ? { files: s.files, servedAgoMs: Date.now() - s.ts } : {}), _hint: s ? 'The intercepted file chooser was satisfied with the staged files.' : (_chooserArm.has(Number(tabId)) ? 'Still armed - click the Browse/Upload control now (nbrowser_click), then re-check.' : 'Not armed and nothing served. Arm with {armForNextChooser:true, filePaths} first.') };
}
if (!files.length) return { ok: false, errorCode: 'no_files', error: 'filePaths is empty', _hint: 'Pass filePaths:["C:/abs/path/on/the/laptop", ...]. Push files to the laptop first with adom-bridge send_files (or use files:[{name,contentBase64}] - the bridge stages them for you).' };
if (args.armForNextChooser === true) {
await send(tabId, 'Page.setInterceptFileChooserDialog', { enabled: true });
const prev = _chooserArm.get(Number(tabId)); if (prev && prev.timer) clearTimeout(prev.timer);
const timer = setTimeout(() => { _chooserArm.delete(Number(tabId)); send(tabId, 'Page.setInterceptFileChooserDialog', { enabled: false }).catch(() => {}); }, 30000);
_chooserArm.set(Number(tabId), { files, timer });
_chooserServed.delete(Number(tabId));
return { ok: true, armed: true, files: files.map((f) => String(f).split(/[\\/]/).pop()), _hint: 'ARMED for 30s: now nbrowser_click the Browse/Upload control - the page\'s file chooser will be satisfied with these files silently (no OS dialog). Then confirm with {chooserStatus:true} or just screenshot the page.' };
}
// direct mode: resolve the input element as a RemoteObject
let obj;
try { obj = await send(tabId, 'Runtime.evaluate', { expression: findFileInputFn(args.selector, args.text), returnByValue: false }); }
catch (e) { return { ok: false, error: 'finder evaluate failed: ' + ((e && e.message) || e) }; }
const ro = obj && obj.result;
if (!ro || !ro.objectId || ro.subtype === 'null') {
const diag = await evaluate(tabId, `(function(){ return { fileInputs: document.querySelectorAll('input[type=file]').length, selMatched: ${args.selector ? `!!document.querySelector(${JSON.stringify(args.selector)})` : 'null'} }; })()`).catch(() => ({}));
return { ok: false, errorCode: 'input_not_found', error: `no <input type=file> resolved from ${args.selector ? 'selector ' + args.selector : args.text ? 'text "' + args.text + '"' : 'the page'}`, diag, _hint: diag && diag.fileInputs === 0 ? 'This page has NO file input right now. If the input is created by the Browse button\'s click handler, use {armForNextChooser:true, filePaths} then click the button. If the site uses showOpenFilePicker (File System Access API), CDP cannot inject files at all - use the site\'s drag-drop zone manually.' : 'A file input exists but your selector/text did not lead to it - target the input directly (selector:"input[type=file]") or the visible Browse control.' };
}
try { await send(tabId, 'DOM.enable').catch(() => {}); await send(tabId, 'DOM.setFileInputFiles', { files, objectId: ro.objectId }); }
catch (e) { return { ok: false, error: 'DOM.setFileInputFiles failed: ' + ((e && e.message) || e), _hint: 'Verify every path exists ON THE LAPTOP (the bridge stats them, but a file may have been moved) and that the resolved element truly is an input[type=file].' }; }
try { await send(tabId, 'Runtime.callFunctionOn', { objectId: ro.objectId, functionDeclaration: `function(){ this.dispatchEvent(new Event('input',{bubbles:true,composed:true})); this.dispatchEvent(new Event('change',{bubbles:true,composed:true})); }` }); } catch {}
const names = await send(tabId, 'Runtime.callFunctionOn', { objectId: ro.objectId, functionDeclaration: 'function(){ return Array.prototype.map.call(this.files||[], function(f){return f.name;}); }', returnByValue: true }).then((r) => (r && r.result && r.result.value) || []).catch(() => []);
return { ok: true, inputFound: true, via: 'direct', files: names, dispatchedEvents: ['input', 'change'], _hint: 'Files injected + input/change dispatched. Screenshot to confirm the page shows the selected file(s); many UIs need a separate Submit/Upload click next.' };
}