← Commit history

v0.0.35 bridge + 0.0.19 ext: shadow-piercing finder, nbrowser_select, screenshot coord-map, session pinning, eager profile-email resolve

John Lauer ·39daf9fa33 ·4mo ago ·parent 916cb57
8 files changed +245−31
bridge/BRIDGE_VERSION+1−1
@@ -1 +1 @@-0.0.34\ No newline at end of file+0.0.35\ No newline at end of file
bridge/bridge.json+10−5
@@ -2,8 +2,8 @@   "manifest_version": 1,   "name": "native-browser",   "displayName": "Adom Native Browser (Chrome + Edge)",-  "version": "0.0.34",-  "expectedExtensionVersion": "0.0.18",+  "version": "0.0.35",+  "expectedExtensionVersion": "0.0.19",   "description": "Drives the user's REAL signed-in Chrome/Edge via an installed extension (native-messaging transport). Mirrors the puppeteer bridge's verb surface as nbrowser_* but runs inside the actual logged-in profile, so cookies/SSO/saved-logins/captcha human-trust all work. The bridge binds an ephemeral loopback port (PORTS.md compliant) and writes a discovery file; the native-messaging host pumps stdio <-> this bridge; the extension never touches a TCP port.",   "homepage": "https://wiki.adom.inc/adom/adom-browser-extension",   "author": "Adom Inc.",@@ -74,7 +74,11 @@     "nbrowser_download_wait",     "nbrowser_totp",     "nbrowser_register_totp",-    "nbrowser_add_extension_origin"+    "nbrowser_add_extension_origin",+    "nbrowser_select",+    "nbrowser_whoami",+    "nbrowser_pin_session",+    "nbrowser_resolve_profiles"   ],   "dependencies": {     "node": ">=18"@@ -109,7 +113,8 @@     "verbs": {       "wait": 125,       "fetch_url": 120,-      "input_dispatch": 120+      "input_dispatch": 120,+      "select": 30     },     "prefixes": [       {@@ -122,4 +127,4 @@       }     ]   }-}+}\ No newline at end of file
bridge/package.json+1−1
@@ -1,6 +1,6 @@ {   "name": "adom-native-browser-bridge",-  "version": "0.0.34",+  "version": "0.0.35",   "private": true,   "description": "adom-desktop bridge for the Adom native-browser extension (Chrome + Edge)",   "main": "server.js",
bridge/server.js+46−3
@@ -124,6 +124,13 @@ function targetConn(args) {     for (const [k, c] of conns) if (c.profile.browser === wantBrowser && c.sock && !c.sock.destroyed && !isBlocked(k)) return { conn: c, key: k };     return null;   }+  // A session PINNED to a profile (via open_window or nbrowser_pin_session) routes there by default -+  // so you can drive several accounts and never accidentally hit the wrong one by forgetting `profile:`.+  const pin = args && args.sessionId && sessionProfile.get(args.sessionId);+  if (pin) {+    for (const [k, c] of conns) if (k === pin && c.sock && !c.sock.destroyed && !isBlocked(k)) return { conn: c, key: k, pinned: true };+    // pinned profile no longer live → fall through to the normal default rather than hard-failing.+  }   const live = liveConns();   if (!live.length) return null;   const act = live.find(([k]) => k === activeProfile);@@ -394,6 +401,7 @@ async function wgcRecord(verb, args) { // (ITaskbarList3 - can't be driven cleanly from PowerShell). See TASKBAR-FEATURE.md. let lastHwnd = 0;                  // most-recently driven window const sessionHwnd = new Map();     // sessionId -> hwnd+const sessionProfile = new Map();  // sessionId -> profile KEY (pin: verbs on this session route here by default) let flashMode = 'quiet';           // quiet | chatty | off const COMELOOK = new Set(['nbrowser_navigate', 'nbrowser_screenshot', 'nbrowser_screenshot_full_res', 'nbrowser_download']); const ACTIVITY = new Set([...COMELOOK, 'nbrowser_reload', 'nbrowser_click', 'nbrowser_double_click', 'nbrowser_right_click', 'nbrowser_type', 'nbrowser_press_key', 'nbrowser_input_dispatch', 'nbrowser_hover', 'nbrowser_eval', 'nbrowser_evaluate', 'nbrowser_open_tab']);@@ -678,7 +686,10 @@ const VERB_META = {   nbrowser_login_state: { hint: 'Is the profile signed in to a URL right now (via chrome.cookies)?', related: ['nbrowser_credentials'], pitfalls: [] },   nbrowser_credentials: { hint: 'Read-only QUERY of the browser\'s own saved-login store (which vendors the user can auto-login to). Never returns passwords.', related: ['nbrowser_login_state', 'nbrowser_navigate'], pitfalls: ['if a vendor is not listed, the user has no saved login - ask them to save one once', 'never exposes the password; Chrome autofills it when you focus the field'] },   nbrowser_profiles: { hint: 'List every BROWSER x PROFILE you can drive + which is active. Drive several in parallel with profile:"<key>".', related: ['nbrowser_use_profile', 'nbrowser_profile_block'], pitfalls: ['active is only the default when no profile is passed'] },-  nbrowser_use_profile: { hint: 'Set the default profile that verbs target.', related: ['nbrowser_profiles'], pitfalls: ['you can still override per-call with profile:"<email>"'] },+  nbrowser_use_profile: { hint: 'Set the default profile that verbs target.', related: ['nbrowser_profiles', 'nbrowser_pin_session'], pitfalls: ['you can still override per-call with profile:"<email>"'] },+  nbrowser_pin_session: { hint: 'Pin a sessionId to a profile so EVERY verb carrying that sessionId auto-routes to it (no cross-profile mistakes when driving several accounts). open_window pins automatically to the profile it opened in; use this to re-pin explicitly.', related: ['nbrowser_profiles', 'nbrowser_resolve_profiles', 'nbrowser_open_window'], pitfalls: ['pins are per-session; a verb with an explicit profile: still overrides the pin', 'if the pinned profile goes offline, verbs fall back to the active profile'] },+  nbrowser_resolve_profiles: { hint: 'Force every connected extension to re-resolve its Google email and re-announce, so pending-N / empty-email keys eagerly become chrome:<email> / edge:<email> - call it up front so you never drive the wrong (unresolved) profile.', related: ['nbrowser_profiles', 'nbrowser_pin_session', 'nbrowser_whoami'], pitfalls: ['takes ~2.5s (waits for each re-announce)', 'a profile not signed into Google stays email-less - target it by id-key or browser:'] },+  nbrowser_whoami: { hint: 'Ask ONE profile to (re)resolve + report its Google email (and re-announce it to the bridge). For all profiles at once use nbrowser_resolve_profiles.', related: ['nbrowser_resolve_profiles', 'nbrowser_profiles'], pitfalls: ['signedIn:false = this browser profile has no Google account, so no email'] },   nbrowser_profile_block: { hint: 'Blocklist a profile so it is NEVER driven (user safety whitelist; persists).', related: ['nbrowser_profile_unblock', 'nbrowser_profiles'], pitfalls: ['respect the blocklist - it is the user telling you hands-off'] },   nbrowser_profile_unblock: { hint: 'Remove a profile from the blocklist.', related: ['nbrowser_profile_block'], pitfalls: [] },   nbrowser_rescan: { hint: 'Reconcile the session map against live windows (drop closed).', related: ['nbrowser_list_windows'], pitfalls: [] },@@ -698,7 +709,8 @@ const VERB_META = {   nbrowser_eval: { hint: 'Run JS in the page and get the result. Powerful - use read-only expressions to inspect state.', related: ['nbrowser_screenshot'], pitfalls: ['runs arbitrary JS on the user\'s LIVE session - never run destructive scripts'] },   nbrowser_evaluate: { hint: 'Alias of nbrowser_eval.', related: ['nbrowser_eval'], pitfalls: ['runs arbitrary JS on the user\'s LIVE session - never run destructive scripts'] },   nbrowser_input_dispatch: { hint: 'Low-level isTrusted CDP input (click/move/type/key).', related: ['nbrowser_click', 'nbrowser_type'], pitfalls: ['prefer nbrowser_click (smart selector) over raw xy'] },-  nbrowser_click: { hint: 'Click by visible text / selector / xy (isTrusted). Scrolls the target into view, re-resolves its rect, and RETURNS changed:bool - whether the URL/DOM actually changed after the click - so you can detect "click landed but nothing happened" and retry. Optional gliding cursor + intent label.', related: ['nbrowser_wait_for', 'nbrowser_screenshot', 'nbrowser_type'], next: 'nbrowser_wait_for', pitfalls: ['attaches the debugger (yellow "debugging" bar; idle-detaches ~8s)', 'changed:false means the click dispatched but the page did not visibly change (disabled/covered element, missed target) - re-resolve or retry', 'for a deterministic post-click wait use nbrowser_wait_for, not a fixed sleep'] },+  nbrowser_click: { hint: 'Click by visible text / selector / xy (isTrusted). Scrolls the target into view, re-resolves its rect, and RETURNS changed:bool - whether the URL/DOM actually changed after the click - so you can detect "click landed but nothing happened" and retry. Optional gliding cursor + intent label. The finder PIERCES open shadow roots, so custom-element controls (Schwab/QBO) are clickable by text/selector.', related: ['nbrowser_select', 'nbrowser_wait_for', 'nbrowser_screenshot', 'nbrowser_type'], next: 'nbrowser_wait_for', pitfalls: ['attaches the debugger (yellow "debugging" bar; idle-detaches ~8s)', 'changed:false means the click dispatched but the page did not visibly change (disabled/covered element, missed target) - re-resolve or retry', 'for a deterministic post-click wait use nbrowser_wait_for, not a fixed sleep', 'to PICK an option from a custom dropdown, use nbrowser_select (waits for options + returns the value) - never coordinate-click a virtualized list'] },+  nbrowser_select: { hint: 'Pick an option from a dropdown/combobox BY ITS VISIBLE TEXT - the reliable primitive for custom + native selects. Give {label} (the control\'s accessible name) OR {near} (a visible label beside it) to find the trigger, plus {optionText}. It opens the control with an isTrusted click, WAITS for the options to render, clicks the EXACT text match (piercing shadow roots), and RETURNS the newly-selected value so you can verify. Fixes coordinate-clicks landing on the wrong row of a virtualized list.', related: ['nbrowser_click', 'nbrowser_wait_for'], pitfalls: ['pass label OR near to locate the trigger; optionText must match the option\'s exact visible text (case-insensitive)', 'ALWAYS check the returned `selected` value before continuing - confirm it is the row you meant (mis-selecting a bank account by one row is the exact bug this prevents)', 'raise timeoutMs for a slow/virtualized option list (default 4000, max 20000)'] },   nbrowser_set_date: { hint: 'Fill a react-aria SEGMENTED date field (role=spinbutton month/day/year, common in finance apps) - focuses it and types each digit as a real keydown so the segments advance. date="YYYY-MM-DD". Pass order:"DMY" for day-first locales (default MDY).', related: ['nbrowser_type', 'nbrowser_click'], pitfalls: ['for react-aria/segmented fields; a plain <input type=date> may just take nbrowser_type', 'US MDY order by default - pass order:"DMY" if the field is day-first', 'check fieldValue in the response; screenshot if it looks off'] },   nbrowser_totp: { hint: 'Type the current 6-digit 2FA code for a host into the FOCUSED field (or pass selector). The bridge computes the RFC-6238 code from the stored seed and the extension types it - the agent NEVER sees the seed or the code. Pass submit:true to press Enter after. Register the seed once with nbrowser_register_totp. The #1 unblock for unattended vendor automation.', related: ['nbrowser_register_totp', 'nbrowser_autologin', 'nbrowser_login_state'], pitfalls: ['errorCode:"no_totp_seed" = register it first with nbrowser_register_totp', 'focus the 2FA input first (click it) or pass selector', 'codes rotate every 30s - call this right when you need it'] },   nbrowser_register_totp: { hint: 'One-time: register a base32 TOTP secret for a host (the "manual entry / setup key" the vendor shows by the 2FA QR). Stored 0600 on the laptop, NEVER returned. Then nbrowser_totp {host} types codes.', related: ['nbrowser_totp'], pitfalls: ['base32 only (A-Z, 2-7)', 'stored on the laptop; the agent never sees it back', 'one seed per host - re-registering replaces it'] },@@ -828,6 +840,9 @@ function dispatch(verb, args) {         d('nbrowser_credentials', "Read-only query of the browser's saved-login store (host+username only; never passwords).", { match: 'optional vendor substring', browser: 'optional "chrome"|"edge"', profile: P }, { ok: 'bool', count: 'int', credentials: '[{browser,profile,host,username}]' }, 20, { example: { match: 'digikey' } }),         d('nbrowser_profiles', 'List connected BROWSER×PROFILE targets + which is active.', {}, { ok: 'bool', profiles: '[{profile,email,browser,active,blocked,live}]', active: 'str' }, 15, { example: {} }),         d('nbrowser_use_profile', 'Set the default profile action verbs target.', { profile: 'required str (key or email)' }, { ok: 'bool', active: 'str' }, 15, { example: { profile: 'chrome:[email protected]' } }),+        d('nbrowser_pin_session', 'Pin a sessionId to a profile so every verb on that session auto-routes there (prevents cross-profile mistakes). open_window pins automatically.', { sessionId: 'required str', profile: 'required str (key or email)' }, { ok: 'bool', sessionId: 'str', pinnedTo: 'str' }, 15, { example: { sessionId: 'schwab', profile: 'chrome:[email protected]' } }),+        d('nbrowser_resolve_profiles', 'Force every connected extension to re-resolve its Google email + re-announce, so pending-N/empty-email keys become chrome:<email>/edge:<email>.', {}, { ok: 'bool', profiles: '[{profile,email,browser}]' }, 15, { example: {} }),+        d('nbrowser_whoami', 'Ask one profile to (re)resolve + report its Google email and re-announce it.', { profile: P }, { ok: 'bool', email: 'str', signedIn: 'bool', browser: 'str' }, 15, { example: {} }),         d('nbrowser_profile_block', 'Blocklist a profile so it is NEVER driven (persists).', { profile: 'required str' }, { ok: 'bool', blocklist: '[str]' }, 15, { example: { profile: 'edge:[email protected]' } }),         d('nbrowser_profile_unblock', 'Remove a profile from the blocklist.', { profile: 'required str' }, { ok: 'bool', blocklist: '[str]' }, 15, { example: { profile: 'edge:[email protected]' } }),         d('nbrowser_rescan', 'Reconcile the session map against live windows (drop closed).', {}, { ok: 'bool', sessions: '[...]' }, 20, { example: {} }),@@ -847,7 +862,8 @@ function dispatch(verb, args) {         d('nbrowser_eval', 'Run JS in the page context and return the result.', { sessionId: S, tabId: 'optional str', expression: 'required str (JS)' }, { ok: 'bool', result: 'any' }, 45, { example: { sessionId: 'work', expression: 'document.title' } }),         d('nbrowser_evaluate', 'Alias of nbrowser_eval.', { sessionId: S, tabId: 'optional str', expression: 'required str (JS)' }, { ok: 'bool', result: 'any' }, 45, { example: { sessionId: 'work', expression: '1+1' } }),         d('nbrowser_input_dispatch', 'Low-level isTrusted CDP input.', { sessionId: S, tabId: 'optional str', type: 'required "click"|"move"|"type"|"key"', x: 'int?', y: 'int?', text: 'str?', key: 'str?' }, { ok: 'bool' }, 120, { example: { sessionId: 'work', type: 'click', x: 200, y: 300 } }),-        d('nbrowser_click', 'Click by visible text / selector / xy; scrolls into view + returns changed:bool (did the URL/DOM actually change). Optional gliding cursor + intent label.', { sessionId: S, tabId: 'optional str', text: 'optional str', selector: 'optional str', x: 'int?', y: 'int?', cursor: 'optional bool', intent: 'optional str', settleMs: 'optional int (default 500)' }, { ok: 'bool', changed: 'bool', urlBefore: 'str', urlAfter: 'str' }, 30, { example: { sessionId: 'work', text: 'Sign in', cursor: true, intent: 'Clicking Sign in' } }),+        d('nbrowser_click', 'Click by visible text / selector / xy; scrolls into view + returns changed:bool (did the URL/DOM actually change). Finder pierces open shadow roots. Optional gliding cursor + intent label.', { sessionId: S, tabId: 'optional str', text: 'optional str', selector: 'optional str', x: 'int?', y: 'int?', cursor: 'optional bool', intent: 'optional str', settleMs: 'optional int (default 500)' }, { ok: 'bool', changed: 'bool', urlBefore: 'str', urlAfter: 'str' }, 30, { example: { sessionId: 'work', text: 'Sign in', cursor: true, intent: 'Clicking Sign in' } }),+        d('nbrowser_select', 'Pick a dropdown/combobox option BY VISIBLE TEXT (custom or native): finds the trigger by label|near, opens it, waits for options, clicks the exact match (shadow-piercing), returns the selected value to verify.', { sessionId: S, tabId: 'optional str', label: 'optional str (control accessible name)', near: 'optional str (visible label beside it)', optionText: 'required str (exact option text)', timeoutMs: 'optional int (default 4000, max 20000)' }, { ok: 'bool', selected: 'str', matchedExact: 'bool', clickedText: 'str', via: '"custom-dropdown"|"native-select"' }, 25, { example: { sessionId: 'work', near: 'Account', optionText: 'Individual …3456' } }),         d('nbrowser_set_date', 'Fill a react-aria segmented date field (role=spinbutton MM/DD/YYYY). Types each digit as a real keydown so segments advance.', { sessionId: S, selector: 'required CSS selector of the date field/segment', date: 'required "YYYY-MM-DD"', order: 'optional "MDY"(default)|"DMY"' }, { ok: 'bool', fieldValue: 'str?' }, 30, { example: { sessionId: 'work', selector: '[aria-label="Start date"]', date: '2026-03-15' } }),         d('nbrowser_totp', 'Type the current 6-digit 2FA code for a host into the focused field (agent never sees seed or code). Register the seed once with nbrowser_register_totp.', { sessionId: S, host: 'required host, e.g. "gusto.com"', selector: 'optional 2FA field selector (else the focused field)', submit: 'optional bool (press Enter after)' }, { ok: 'bool', typed: 'bool', digits: 'int' }, 20, { example: { sessionId: 'work', host: 'gusto.com', submit: true } }),         d('nbrowser_register_totp', 'One-time: register a base32 TOTP secret for a host (stored 0600 on the laptop, never returned).', { host: 'required host', seed: 'required base32 secret' }, { ok: 'bool', host: 'str' }, 15, { example: { host: 'gusto.com', seed: 'JBSWY3DPEHPK3PXP' } }),@@ -883,6 +899,30 @@ function dispatch(verb, args) {       activeProfile = hit;       return resolve({ success: true, output: JSON.stringify({ ok: true, active: activeProfile, _hint: 'Action verbs now default to this profile. You can still override per-call with profile:"<email>".' }) });     }+    // PIN a session to a profile: every later verb carrying this sessionId auto-routes there (without+    // needing profile: on each call), so driving several accounts can't cross wires. open_window pins+    // automatically; this is the explicit/re-pin path.+    if (verb === 'nbrowser_pin_session') {+      const sid = args && args.sessionId;+      const want = String((args && (args.profile || args.email)) || '');+      if (!sid) return resolve({ success: false, error: 'sessionId required', _hint: 'nbrowser_pin_session {sessionId, profile} pins that session to a profile so later verbs on it auto-route there.' });+      const hit = [...conns.keys()].find((k) => k === want || (conns.get(k).profile.email === want));+      if (!hit) return resolve({ success: false, error: `no connected profile matches '${want}'`, _hint: 'Run nbrowser_profiles to list connected profiles by key/email.' });+      sessionProfile.set(sid, hit);+      return resolve({ success: true, output: JSON.stringify({ ok: true, sessionId: sid, pinnedTo: hit, _hint: `Session "${sid}" is pinned to ${hit}. Any verb carrying this sessionId (without an explicit profile:) now routes there. Re-pin anytime.` }) });+    }+    // Force every connected extension to RE-RESOLVE its Google email and re-announce, so pending-N/id keys+    // eagerly become chrome:<email>/edge:<email> - up front, before you drive and risk the wrong profile.+    if (verb === 'nbrowser_resolve_profiles') {+      const live = liveConns();+      if (!live.length) return resolve({ success: true, output: JSON.stringify({ ok: true, profiles: [], _hint: 'No profiles connected yet.' }) });+      for (const [, c] of live) { try { const id = ++seq; pending.set(id, { resolve: () => {}, timer: setTimeout(() => pending.delete(id), 4000), verb: 'nbrowser_whoami' }); c.sock.write(JSON.stringify({ id, verb: 'nbrowser_whoami', args: {} }) + '\n'); } catch {} }+      setTimeout(() => {+        const list = [...conns.entries()].filter(([, c]) => c.sock && !c.sock.destroyed).map(([k, c]) => ({ profile: k, email: c.profile.email || '', browser: c.profile.browser || 'chrome' }));+        resolve({ success: true, output: JSON.stringify({ ok: true, profiles: list, _hint: 'Re-resolved each connected extension\'s Google email + re-announced; keys now reflect chrome:<email> / edge:<email> where signed in. A profile with an empty email is not signed into Google - target it by its id-key or by browser:.' }) });+      }, 2500);+      return;+    }     if (verb === 'nbrowser_profile_block' || verb === 'nbrowser_profile_unblock') {       const want = String((args && (args.profile || args.email)) || '');       if (!want) return resolve({ success: false, error: 'profile (email) required' });@@ -1058,6 +1098,9 @@ function dispatch(verb, args) {       pending.set(id, { resolve, timer, verb, toFile: !!(args && args.toFile) });       sock.write(JSON.stringify({ id, verb, args: args || {} }) + '\n');     };+    // PIN the session to whatever profile this window opened in, so every later verb carrying this+    // sessionId auto-routes back to the SAME profile (no cross-profile mistakes) even without `profile:`.+    if (verb === 'nbrowser_open_window' && args && args.sessionId) sessionProfile.set(args.sessionId, tgt.key);     // For open_window, snapshot the Chrome window list BEFORE forwarding so the post-open diff reliably     // catches the new window (the old fire-and-forget snapshot raced the window appearing).     if (verb === 'nbrowser_open_window') chromeHwnds().then((s) => { preOpenHwnds = s; forward(); }).catch(forward);
extension/manifest.json+1−1
@@ -2,7 +2,7 @@   "manifest_version": 3,   "name": "Adom for Chrome & Edge",   "short_name": "Adom",-  "version": "0.0.18",+  "version": "0.0.19",   "description": "Drive your real, logged-in Chrome/Edge from the Adom ecosystem \u2014 the native-browser counterpart to pup.",   "minimum_chrome_version": "116",   "background": {
extension/src/cdp.js+139−6
@@ -65,9 +65,31 @@ export async function screenshot(tabId, { fullPage = false, maxWidth = 1568, res   const res = await send(tabId, 'Page.captureScreenshot', params);   bumpIdle(tabId);   const out = resize ? await resizePng(res.data, maxWidth) : await pngDims(res.data);-  return { ok: true, base64: out.base64, width: out.width, height: out.height,-    origWidth: out.origWidth ?? out.width, origHeight: out.origHeight ?? out.height,-    resized: !!out.resized, sizeKB: Math.round((out.base64.length * 3 / 4) / 1024) };+  const iw = out.width, ih = out.height, ow = out.origWidth ?? out.width, oh = out.origHeight ?? out.height;+  // Viewport coordinate-map, sampled AT capture time, so screenshot-pixel -> click-coordinate mapping is+  // exact and never needs a separate probe (and can't drift from a later scroll). To turn a pixel (sx,sy)+  // in the RETURNED image into a viewport click coordinate (CSS px, what nbrowser_click/input_dispatch+  // take): clickX = sx * cssPerImageX, clickY = sy * cssPerImageY. scrollX/scrollY are included for+  // page-absolute math; devicePixelRatio + captured/image dims are given so any custom mapping is possible.+  let viewport = null;+  try {+    const m = await evaluate(tabId, '({iw:innerWidth,ih:innerHeight,dpr:devicePixelRatio,sx:Math.round(scrollX),sy:Math.round(scrollY),dw:Math.round(document.documentElement.scrollWidth),dh:Math.round(document.documentElement.scrollHeight)})');+    if (m && typeof m === 'object') {+      const spanW = fullPage ? m.dw : m.iw, spanH = fullPage ? m.dh : m.ih;   // CSS px the image spans+      viewport = {+        innerWidth: m.iw, innerHeight: m.ih, devicePixelRatio: m.dpr,+        scrollX: m.sx, scrollY: m.sy,+        capturedWidth: ow, capturedHeight: oh,       // raw CDP capture px (~ span*dpr)+        imageWidth: iw, imageHeight: ih,             // the returned (possibly downscaled) image px+        resizeRatio: ow ? +(iw / ow).toFixed(6) : 1,+        fullPage: !!fullPage,                        // true = image spans the whole page; coords are page-absolute+        cssPerImageX: iw ? +(spanW / iw).toFixed(6) : 1,   // <- multiply a screenshot X by this to get a click X (CSS px)+        cssPerImageY: ih ? +(spanH / ih).toFixed(6) : 1,+      };+    }+  } catch (e) {}+  return { ok: true, base64: out.base64, width: iw, height: ih, origWidth: ow, origHeight: oh,+    resized: !!out.resized, sizeKB: Math.round((out.base64.length * 3 / 4) / 1024), viewport }; }  // ---- nbrowser_eval / nbrowser_evaluate ------------------------------------@@ -233,8 +255,21 @@ function phaseLabel(type, tgt, phase) { // so {debug:true} never regresses a click. The numeric `score` is for human/AI debugging only. function findFn(query, limit, rich) {   var q = (query || '').toLowerCase();-  var sel = 'a,button,[role=button],[role=link],[role=menuitem],[role=tab],[role=checkbox],input,select,textarea,summary,[onclick],[tabindex]';-  var els = [].slice.call(document.querySelectorAll(sel));+  var sel = 'a,button,[role=button],[role=link],[role=menuitem],[role=menuitemradio],[role=menuitemcheckbox],[role=tab],[role=checkbox],[role=radio],[role=switch],[role=option],[role=combobox],[role=treeitem],[role=gridcell],input,select,textarea,summary,[onclick],[tabindex],[contenteditable]';+  // Deep collector: light DOM + every OPEN shadow root (recursively). Custom-element finance/SPA+  // widgets (Schwab account picker, QBO report search) put their real options INSIDE shadow roots that+  // document.querySelectorAll can't see - so we descend into each element's .shadowRoot. (Closed shadow+  // roots are invisible to any script; only open ones are reachable, which covers the vast majority.)+  function deepQ(root, s) {+    var acc = [];+    (function walk(node) {+      try { acc.push.apply(acc, node.querySelectorAll(s)); } catch (e) {}+      var kids; try { kids = node.querySelectorAll('*'); } catch (e) { kids = []; }+      for (var i = 0; i < kids.length; i++) { var sr = kids[i].shadowRoot; if (sr) walk(sr); }+    })(root || document);+    return acc;+  }+  var els = deepQ(document, sel);   function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01 && r.bottom > 0 && r.right > 0 && r.top < innerHeight && r.left < innerWidth; }   function txt(el) { return ((el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.getAttribute('title') || '') + '').replace(/\s+/g, ' ').trim(); }   function cssPath(el) { try { if (!el || el.nodeType !== 1) return ''; if (el.id) return '#' + el.id; var p = el.tagName.toLowerCase(); var cn = el.className; cn = (cn && cn.baseVal !== undefined ? cn.baseVal : cn) || ''; var cls = ('' + cn).trim().split(/\s+/).filter(Boolean).slice(0, 2); if (cls.length) p += '.' + cls.join('.'); var par = el.parentElement; if (par) { var same = [].slice.call(par.children).filter(function (x) { return x.tagName === el.tagName; }); if (same.length > 1) p += ':nth-of-type(' + (same.indexOf(el) + 1) + ')'; } return p; } catch (e) { return ''; } }@@ -262,7 +297,7 @@ function findFn(query, limit, rich) {   // take the smallest (leaf-most) visible element whose text contains the query — that's the   // specific thing the user named, not a giant wrapping container.   if (q && out.length === 0) {-    var all = [].slice.call(document.querySelectorAll('body *'));+    var all = deepQ(document, '*');     var fb = [];     for (var j = 0; j < all.length; j++) {       var e2 = all[j]; if (!vis(e2)) continue;@@ -289,7 +324,23 @@ export async function find(tabId, query, limit, rich) {  // All visible matches of a CSS selector, with debug data — for {debug}/{resolveOnly} on selector targets. function selectorCandidatesFn(selector) {+  // Light-DOM first (fast, honors combinators). If it finds nothing, deep-walk every open shadow root and+  // match each element with .matches(selector) - so a stable attribute (e.g. [data-testid=x]) or class/tag+  // selector pierces custom-element shadow DOM. (Cross-shadow combinators like ".a > .b" only match within+  // one root; simple attribute/class/tag/role selectors are what this recovers, which is the common case.)   var els = [].slice.call(document.querySelectorAll(selector));+  if (!els.length) {+    var deep = [];+    (function walk(node) {+      var kids; try { kids = node.querySelectorAll('*'); } catch (e) { kids = []; }+      for (var i = 0; i < kids.length; i++) {+        var el = kids[i];+        try { if (el.matches && el.matches(selector)) deep.push(el); } catch (e) {}+        if (el.shadowRoot) walk(el.shadowRoot);+      }+    })(document);+    els = deep;+  }   function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01; }   function cssPath(el) { try { if (el.id) return '#' + el.id; var p = el.tagName.toLowerCase(); var par = el.parentElement; if (par) { var same = [].slice.call(par.children).filter(function (x) { return x.tagName === el.tagName; }); if (same.length > 1) p += ':nth-of-type(' + (same.indexOf(el) + 1) + ')'; } return p; } catch (e) { return ''; } }   var c = els.filter(vis); if (!c.length) c = els;@@ -304,6 +355,88 @@ function selectorCandidatesFn(selector) {   }); } +// ---- nbrowser_select: open a custom dropdown and pick an option by its VISIBLE TEXT ----------------+// Solves: coordinate clicks on virtualized/custom option lists land on the WRONG row (mis-selecting a+// bank account by one row). This locates the trigger (by accessible `label` or the `near` text beside+// it), opens it with an isTrusted CDP click, WAITS for the options to render, clicks the EXACT text+// match (shadow-piercing), and returns the newly-displayed value so the caller can verify.+function selectTriggerFn(label, near) {+  function deepQ(root, s) { var a = []; (function w(n) { try { a.push.apply(a, n.querySelectorAll(s)); } catch (e) {} var k; try { k = n.querySelectorAll('*'); } catch (e) { k = []; } for (var i = 0; i < k.length; i++) { var sr = k[i].shadowRoot; if (sr) w(sr); } })(root || document); return a; }+  function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01 && r.top < innerHeight && r.bottom > 0; }+  function txt(el) { return (((el.getAttribute && (el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('placeholder'))) || el.innerText || el.textContent || '') + '').replace(/\s+/g, ' ').trim(); }+  var cand = null;+  if (label) {+    var lab = (label + '').toLowerCase();+    var trigs = deepQ(document, 'select,[role=combobox],[role=listbox],[aria-haspopup],button,[role=button],input').filter(vis);+    trigs.sort(function (a, b) { function sc(e) { var t = txt(e).toLowerCase(); return t === lab ? 3 : t.indexOf(lab) === 0 ? 2 : t.indexOf(lab) >= 0 ? 1 : 0; } return sc(b) - sc(a); });+    if (trigs.length && txt(trigs[0]).toLowerCase().indexOf(lab) >= 0) cand = trigs[0];+  }+  if (!cand && near) {+    var nl = (near + '').toLowerCase();+    var texts = deepQ(document, 'label,span,div,legend,th,td,p').filter(function (e) { return vis(e) && txt(e).toLowerCase().indexOf(nl) >= 0; });+    texts.sort(function (a, b) { return txt(a).length - txt(b).length; });+    if (texts.length) {+      var L = texts[0], forId = L.getAttribute && L.getAttribute('for');+      if (forId) { var t = document.getElementById(forId); if (t && vis(t)) cand = t; }+      if (!cand) {+        var lr = L.getBoundingClientRect(), lx = lr.left + lr.width / 2, ly = lr.top + lr.height / 2;+        var ctrls = deepQ(document, 'select,[role=combobox],[aria-haspopup],button,[role=button],input,[role=listbox]').filter(vis);+        ctrls.sort(function (a, b) { var ra = a.getBoundingClientRect(), rb = b.getBoundingClientRect(); return Math.hypot(ra.left + ra.width / 2 - lx, ra.top + ra.height / 2 - ly) - Math.hypot(rb.left + rb.width / 2 - lx, rb.top + rb.height / 2 - ly); });+        if (ctrls.length) cand = ctrls[0];+      }+    }+  }+  if (!cand) return { found: false };+  var r = cand.getBoundingClientRect();+  return { found: true, x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), tag: cand.tagName.toLowerCase(), isNativeSelect: cand.tagName.toLowerCase() === 'select', current: txt(cand).slice(0, 120) };+}+function nativeSetFn(x, y, optionText) {+  var el = document.elementFromPoint(x, y); while (el && el.tagName !== 'SELECT') el = el.parentElement;+  if (!el) return { ok: false, error: 'native select not found at point' };+  var ot = (optionText + '').toLowerCase(), opts = [].slice.call(el.options), m = null;+  for (var i = 0; i < opts.length; i++) { if ((opts[i].text || '').replace(/\s+/g, ' ').trim().toLowerCase() === ot) { m = opts[i]; break; } }+  if (!m) for (var j = 0; j < opts.length; j++) { if ((opts[j].text || '').toLowerCase().indexOf(ot) >= 0) { m = opts[j]; break; } }+  if (!m) return { ok: false, error: 'option not found in select: ' + optionText };+  el.value = m.value; el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true }));+  return { ok: true, selected: (m.text || '').replace(/\s+/g, ' ').trim() };+}+function findOptionFn(optionText) {+  function deepQ(root, s) { var a = []; (function w(n) { try { a.push.apply(a, n.querySelectorAll(s)); } catch (e) {} var k; try { k = n.querySelectorAll('*'); } catch (e) { k = []; } for (var i = 0; i < k.length; i++) { var sr = k[i].shadowRoot; if (sr) w(sr); } })(root || document); return a; }+  function txt(e) { return ((e.innerText || e.textContent || '') + '').replace(/\s+/g, ' ').trim(); }+  var ot = (optionText + '').toLowerCase();+  var opts = deepQ(document, '[role=option],[role=menuitem],[role=menuitemradio],[role=treeitem],[role=row],li,option,a,button').filter(function (e) { var r = e.getBoundingClientRect(), s = getComputedStyle(e); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01; });+  var exact = opts.filter(function (e) { return txt(e).toLowerCase() === ot; });+  var cont = exact.length ? exact : opts.filter(function (e) { return txt(e).toLowerCase().indexOf(ot) >= 0; });+  if (!cont.length) return { found: false, optionCount: opts.length };+  cont.sort(function (a, b) { var ra = a.getBoundingClientRect(), rb = b.getBoundingClientRect(); return (ra.width * ra.height) - (rb.width * rb.height); }); // smallest = leaf-most option row+  var el = cont[0], r = el.getBoundingClientRect();+  return { found: true, x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), text: txt(el).slice(0, 120), exact: exact.length > 0, optionCount: opts.length };+}+export async function selectOption(tabId, args) {+  await attach(tabId);+  const timeoutMs = Math.min(Math.max(Number(args.timeoutMs) || 4000, 500), 20000);+  const inj = (func, a) => chrome.scripting.executeScript({ target: { tabId }, func, args: a }).then((r) => r && r[0] && r[0].result);+  const t = await inj(selectTriggerFn, [args.label || '', args.near || '']);+  if (!t || !t.found) return { ok: false, error: 'dropdown trigger not found' + (args.label ? ` for label "${args.label}"` : '') + (args.near ? ` near "${args.near}"` : ''), _hint: 'pass label (the control\'s accessible name / visible text) or near (a visible text label beside it)' };+  if (t.isNativeSelect) {+    const r = await inj(nativeSetFn, [t.x, t.y, args.optionText || '']);+    return r && r.ok ? { ok: true, selected: r.selected, via: 'native-select', matchedExact: true, _hint: 'set a native <select> by option text and dispatched change.' } : { ok: false, error: (r && r.error) || 'native select failed' };+  }+  await mouseClick(tabId, t.x, t.y);   // open (isTrusted)+  let opt = null; const deadline = Date.now() + timeoutMs; let lastCount = 0;+  while (Date.now() < deadline) {+    await new Promise((r) => setTimeout(r, 150));+    const o = await inj(findOptionFn, [args.optionText || '']);+    if (o) { lastCount = o.optionCount || lastCount; if (o.found) { opt = o; break; } }+  }+  if (!opt) return { ok: false, error: `option "${args.optionText}" did not appear after opening the dropdown (${lastCount} option(s) seen)`, _hint: 'the trigger opened but no option matched that EXACT visible text; check spelling/case, or raise timeoutMs for a slow/virtualized list' };+  await mouseClick(tabId, opt.x, opt.y);   // pick (isTrusted)+  await new Promise((r) => setTimeout(r, 220));+  const after = await inj(selectTriggerFn, [args.label || '', args.near || '']);+  const selected = (after && after.found && after.current) ? after.current : opt.text;+  return { ok: true, selected, matchedExact: opt.exact, clickedText: opt.text, via: 'custom-dropdown', _hint: `selected "${selected}". Verify it matches what you intended before continuing.` };+}+ // ---- annotated overlay shot: draw every candidate's rect (numbered), chosen highlighted, click crosshair. // Injected into the page, CDP-screenshotted, then removed — so the overlay never lingers. Rects are // viewport-relative (getBoundingClientRect), and we capture the viewport, so fixed-positioned boxes align.
extension/src/sw.js+43−10
@@ -71,24 +71,44 @@ function connect() { // ([email protected], [email protected], …) and the AI can target/switch between them. Email comes from // chrome.identity; if the profile isn't signed into Google, fall back to a stable random id. let helloSent = false;-async function sendHello() {-  const profile = { email: '', id: '', name: '' };-  // Stable id FIRST — always available, so the hello can ALWAYS be sent even if identity misbehaves.+let _emailResolved = false, _emailTries = 0;+async function stableId() {   try {     const got = await chrome.storage.local.get('_nbProfileId');     let pid = got && got._nbProfileId;     if (!pid) { pid = 'p_' + Math.random().toString(36).slice(2, 10); await chrome.storage.local.set({ _nbProfileId: pid }); }-    profile.id = pid;-  } catch {}-  // Enrich with the Google email if identity cooperates — timeout-protected (can hang with no throw).+    return pid;+  } catch { return 'p_unknown'; }+}+async function resolveEmail() {   try {     const info = await Promise.race([       new Promise((res) => { try { chrome.identity.getProfileUserInfo({ accountStatus: 'ANY' }, res); } catch { res(null); } }),-      new Promise((res) => setTimeout(() => res(null), 1200)),+      new Promise((res) => setTimeout(() => res(null), 1500)),     ]);-    if (info && info.email) profile.email = info.email;-  } catch {}-  try { if (port) { port.postMessage({ type: 'hello', profile, version: VERSION, browser: detectBrowser() }); helloSent = true; } } catch {}+    return (info && info.email) || '';+  } catch { return ''; }+}+function postHello(profile) { try { if (port) { port.postMessage({ type: 'hello', profile, version: VERSION, browser: detectBrowser() }); helloSent = true; } } catch {} }+async function sendHello() {+  // Stable id FIRST — always available, so the hello can ALWAYS be sent even if identity misbehaves.+  const profile = { email: await resolveEmail(), id: await stableId(), name: '' };+  if (profile.email) _emailResolved = true;+  postHello(profile);+  // If the email didn't resolve (identity slow, or briefly signed-out), retry and RE-announce once it+  // lands - so the bridge re-keys "pending/id" -> the real "chrome:[email protected]" WITHOUT the caller+  // doing anything. This is what removes the "pending-N / empty email until driven" gap.+  if (!_emailResolved) scheduleEmailRetry();+}+function scheduleEmailRetry() {+  if (_emailResolved || _emailTries >= 6) return;+  _emailTries++;+  setTimeout(async () => {+    if (_emailResolved) return;+    const email = await resolveEmail();+    if (email) { _emailResolved = true; postHello({ email, id: await stableId(), name: '' }); }+    else scheduleEmailRetry();+  }, Math.min(1500 * _emailTries, 10000)); }  function reply(id, ok, payload, error, hint) {@@ -114,6 +134,17 @@ async function dispatch(verb, args) {     // --- meta -------------------------------------------------------------     case 'nbrowser_status':       return { ok: true, ext: 'adom-browser-extension', version: VERSION, engine: 'native', browser: detectBrowser() };+    case 'nbrowser_whoami': {+      // Force a fresh Google-email resolve + RE-announce, so the bridge re-keys this connection to+      // "chrome:[email protected]" eagerly (fixes "pending-N / empty email until driven"). Returns the email.+      _emailResolved = false; _emailTries = 0;+      const email = await resolveEmail();+      if (email) _emailResolved = true;+      const id = await stableId();+      postHello({ email, id, name: '' });+      return { ok: true, email, id, browser: detectBrowser(), signedIn: !!email,+        _hint: email ? 'Resolved this profile\'s Google email and re-announced it to the bridge.' : 'This browser profile is NOT signed into Google, so it has no email - target it by its id-key or by browser:.' };+    }     case 'nbrowser_ping':       return { ok: true, pong: true };     case 'nbrowser_dev_reload': {@@ -175,6 +206,8 @@ async function dispatch(verb, args) {       return cdp.inputDispatch(await native.resolveTabId(args), args);     case 'nbrowser_click':       return cdp.clickChanged(await native.resolveTabId(args), args); // #5 robust: reports changed:bool+    case 'nbrowser_select': // open a custom/native dropdown, pick option by VISIBLE TEXT, return selected value+      return cdp.selectOption(await native.resolveTabId(args), args);     case 'nbrowser_set_date':      // #7 react-aria segmented date field       return cdp.setDate(await native.resolveTabId(args), args);     case 'nbrowser_totp':          // #3 type the bridge-computed 6-digit code (args.__code) into the focused 2FA field
native-browser-bridge-manifest.json+4−4
@@ -1,8 +1,8 @@ {   "manifest_version": 1,-  "version": "0.0.34",-  "url": "native-browser-bridge-v0.0.34.bin",-  "sha256": "e886ae1e711f28e942bb1eb061ec98753087b667592c19f7b06c37974ca511e8",-  "size": 41208,+  "version": "0.0.35",+  "url": "native-browser-bridge-v0.0.35.bin",+  "sha256": "d8bb45d1c1932f028ce8a304258cbb713a4b25c25dc74088391ae5e5c58538a5",+  "size": 43378,   "released_at": "2026-07-06T00:00:00Z" }\ No newline at end of file