NB - Native Browser Bridge

@@ -202,18 +202,34 @@ function phaseLabel(type, tgt, phase) {  // ---- smart finder: locate visible interactive elements (for the screenshot->find->click loop) ----- // Injected via chrome.scripting (no debugger banner). Matches by visible text / aria-label.-function findFn(query, limit) {+// `rich`: when true, attach per-candidate debug data (score/scoreBreakdown, occlusion, visibility,+// inViewport, a best-effort CSS selector) WITHOUT changing the selection order — chosen stays index 0,+// so {debug:true} never regresses a click. The numeric `score` is for human/AI debugging only.+function findFn(query, limit, rich) {   var q = (query || '').toLowerCase();   var sel = 'a,button,[role=button],[role=link],[role=menuitem],[role=tab],[role=checkbox],input,select,textarea,summary,[onclick],[tabindex]';   var els = [].slice.call(document.querySelectorAll(sel));   function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 1 && r.height > 1 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01 && r.bottom > 0 && r.right > 0 && r.top < innerHeight && r.left < innerWidth; }   function txt(el) { return ((el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.getAttribute('title') || '') + '').replace(/\s+/g, ' ').trim(); }+  function cssPath(el) { try { if (!el || el.nodeType !== 1) return ''; if (el.id) return '#' + el.id; var p = el.tagName.toLowerCase(); var cn = el.className; cn = (cn && cn.baseVal !== undefined ? cn.baseVal : cn) || ''; var cls = ('' + cn).trim().split(/\s+/).filter(Boolean).slice(0, 2); if (cls.length) p += '.' + cls.join('.'); var par = el.parentElement; if (par) { var same = [].slice.call(par.children).filter(function (x) { return x.tagName === el.tagName; }); if (same.length > 1) p += ':nth-of-type(' + (same.indexOf(el) + 1) + ')'; } return p; } catch (e) { return ''; } }+  function richOf(el, t, al) {+    var r = el.getBoundingClientRect(); var tl = (t || '').toLowerCase(); var tag = el.tagName.toLowerCase();+    var textMatch = !q ? 0 : tl === q ? 100 : al === q ? 95 : tl.indexOf(q) === 0 ? 70 : tl.indexOf(q) >= 0 ? 50 : al.indexOf(q) >= 0 ? 40 : 0;+    var clickableTag = (/^(a|button|input|select|textarea|summary)$/.test(tag) || el.getAttribute('role')) ? 20 : 0;+    var visible = vis(el); var visibility = visible ? 10 : 0;+    var inViewport = r.top >= 0 && r.left >= 0 && r.bottom <= (innerHeight || 0) && r.right <= (innerWidth || 0);+    var cx = Math.round(r.left + r.width / 2), cy = Math.round(r.top + r.height / 2);+    var occluded = false, occludedBy = '';+    try { var top = document.elementFromPoint(cx, cy); if (top && top !== el && !el.contains(top) && !top.contains(el)) { occluded = true; occludedBy = cssPath(top); } } catch (e) {}+    var occlusionPenalty = occluded ? -30 : 0;+    return { score: textMatch + clickableTag + visibility + occlusionPenalty, scoreBreakdown: { textMatch: textMatch, clickableTag: clickableTag, visibility: visibility, occlusionPenalty: occlusionPenalty, area: Math.round(r.width * r.height) }, visible: visible, occluded: occluded, occludedBy: occludedBy, inViewport: inViewport, selector: cssPath(el) };+  }+  function mk(el, t, r) { var o = { text: (t || '').slice(0, 80), tag: el.tagName.toLowerCase(), role: el.getAttribute('role') || '', type: el.type || '', x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), rect: { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height), centerX: Math.round(r.left + r.width / 2), centerY: Math.round(r.top + r.height / 2) } }; if (rich) { var rf = richOf(el, t, (el.getAttribute('aria-label') || '').toLowerCase()); for (var k in rf) o[k] = rf[k]; } return o; }   var out = [];   for (var i = 0; i < els.length; i++) {     var el = els[i]; if (!vis(el)) continue; var t = txt(el); var al = (el.getAttribute('aria-label') || '').toLowerCase();     if (q && t.toLowerCase().indexOf(q) < 0 && al.indexOf(q) < 0) continue;-    var r = el.getBoundingClientRect();-    out.push({ text: t.slice(0, 80), tag: el.tagName.toLowerCase(), role: el.getAttribute('role') || '', type: el.type || '', x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), rect: { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height) } });+    out.push(mk(el, t, el.getBoundingClientRect()));   }   // Fallback: hover/move targets are often NON-interactive (info icons, cards, tooltip   // triggers, plain spans). If nothing interactive matched the text, scan all elements and@@ -227,7 +243,7 @@ function findFn(query, limit) {       var t2 = ((e2.innerText || e2.textContent || '') + '').replace(/\s+/g, ' ').trim();       if (!t2 || t2.toLowerCase().indexOf(q) < 0) continue;       var r2 = e2.getBoundingClientRect();-      fb.push({ text: t2.slice(0, 80), tag: e2.tagName.toLowerCase(), role: e2.getAttribute('role') || '', type: '', area: r2.width * r2.height, x: Math.round(r2.left + r2.width / 2), y: Math.round(r2.top + r2.height / 2), rect: { x: Math.round(r2.left), y: Math.round(r2.top), w: Math.round(r2.width), h: Math.round(r2.height) } });+      var o2 = mk(e2, t2, r2); o2.area = r2.width * r2.height; fb.push(o2);     }     // exact text match wins, else smallest area (most specific leaf) — return directly,     // skipping the interactive-pass sort below (which prefers LARGEST, wrong for leaves).@@ -237,27 +253,103 @@ function findFn(query, limit) {   out.sort(function (a, b) { var ae = a.text.toLowerCase() === q ? 0 : 1, be = b.text.toLowerCase() === q ? 0 : 1; if (ae !== be) return ae - be; return (b.rect.w * b.rect.h) - (a.rect.w * a.rect.h); });   return out.slice(0, limit || 40); }-export async function find(tabId, query, limit) {+export async function find(tabId, query, limit, rich) {   try {-    const r = await chrome.scripting.executeScript({ target: { tabId }, func: findFn, args: [query || '', limit || 40] });+    const r = await chrome.scripting.executeScript({ target: { tabId }, func: findFn, args: [query || '', limit || 40, !!rich] });     const els = (r && r[0] && r[0].result) || [];     return { ok: true, query: query || '', count: els.length, elements: els };   } catch (e) { return { ok: false, error: String((e && e.message) || e) }; } } -async function resolveCoords(tabId, args) {+// All visible matches of a CSS selector, with debug data — for {debug}/{resolveOnly} on selector targets.+function selectorCandidatesFn(selector) {+  var els = [].slice.call(document.querySelectorAll(selector));+  function vis(el) { var r = el.getBoundingClientRect(), s = getComputedStyle(el); return r.width > 0 && r.height > 0 && s.visibility !== 'hidden' && s.display !== 'none' && parseFloat(s.opacity || '1') > 0.01; }+  function cssPath(el) { try { if (el.id) return '#' + el.id; var p = el.tagName.toLowerCase(); var par = el.parentElement; if (par) { var same = [].slice.call(par.children).filter(function (x) { return x.tagName === el.tagName; }); if (same.length > 1) p += ':nth-of-type(' + (same.indexOf(el) + 1) + ')'; } return p; } catch (e) { return ''; } }+  var c = els.filter(vis); if (!c.length) c = els;+  var modal = document.querySelector('dialog[open],[aria-modal="true"],[role="dialog"]');+  if (modal) { var m = c.filter(function (e) { return modal.contains(e); }); if (m.length) c = m; }+  if (!c.length) return [];+  var chosenEl = c.reduce(function (a, b) { var ra = a.getBoundingClientRect(), rb = b.getBoundingClientRect(); return (rb.width * rb.height) > (ra.width * ra.height) ? b : a; });+  return c.slice(0, 40).map(function (el) {+    var r = el.getBoundingClientRect(); var cx = Math.round(r.left + r.width / 2), cy = Math.round(r.top + r.height / 2);+    var occ = false, occBy = ''; try { var top = document.elementFromPoint(cx, cy); if (top && top !== el && !el.contains(top) && !top.contains(el)) { occ = true; occBy = cssPath(top); } } catch (e) {}+    return { text: ((el.innerText || el.value || '') + '').replace(/\s+/g, ' ').trim().slice(0, 80), tag: el.tagName.toLowerCase(), role: el.getAttribute('role') || '', selector: cssPath(el), x: cx, y: cy, rect: { x: Math.round(r.left), y: Math.round(r.top), w: Math.round(r.width), h: Math.round(r.height), centerX: cx, centerY: cy }, score: el === chosenEl ? 100 : 50, scoreBreakdown: { rule: 'largest-visible-match', area: Math.round(r.width * r.height) }, visible: vis(el), occluded: occ, occludedBy: occBy, inViewport: r.top >= 0 && r.left >= 0 && r.bottom <= innerHeight && r.right <= innerWidth, chosen: el === chosenEl };+  });+}++// ---- annotated overlay shot: draw every candidate's rect (numbered), chosen highlighted, click crosshair.+// Injected into the page, CDP-screenshotted, then removed — so the overlay never lingers. Rects are+// viewport-relative (getBoundingClientRect), and we capture the viewport, so fixed-positioned boxes align.+function overlayDrawFn(boxes, chosenIndex, clickX, clickY) {+  var id = '__adom_finder_overlay__'; var prev = document.getElementById(id); if (prev) prev.remove();+  var o = document.createElement('div'); o.id = id; o.style.cssText = 'position:fixed;inset:0;z-index:2147483646;pointer-events:none;margin:0;padding:0;';+  for (var i = 0; i < boxes.length; i++) {+    var c = boxes[i], r = c.rect, chosen = (i === chosenIndex);+    var col = chosen ? '#00e676' : (c.occluded ? '#ff5252' : '#ffb300');+    var b = document.createElement('div');+    b.style.cssText = 'position:fixed;left:' + r.x + 'px;top:' + r.y + 'px;width:' + r.w + 'px;height:' + r.h + 'px;border:' + (chosen ? '3px' : '2px') + ' solid ' + col + ';box-sizing:border-box;background:' + (chosen ? 'rgba(0,230,118,0.14)' : 'transparent') + ';';+    var lab = document.createElement('div'); lab.textContent = i + (chosen ? ' ★' : '') + (c.occluded ? ' ⛔' : '');+    lab.style.cssText = 'position:absolute;top:-15px;left:0;font:bold 12px/14px monospace;color:#000;background:' + col + ';padding:0 4px;border-radius:3px;white-space:nowrap;';+    b.appendChild(lab); o.appendChild(b);+  }+  if (clickX != null && clickY != null) {+    var ch = document.createElement('div'); ch.style.cssText = 'position:fixed;left:' + (clickX - 11) + 'px;top:' + (clickY - 11) + 'px;width:22px;height:22px;pointer-events:none;';+    ch.innerHTML = '<svg width="22" height="22"><line x1="11" y1="0" x2="11" y2="22" stroke="#2979ff" stroke-width="2"/><line x1="0" y1="11" x2="22" y2="11" stroke="#2979ff" stroke-width="2"/><circle cx="11" cy="11" r="7" fill="none" stroke="#2979ff" stroke-width="2"/></svg>';+    o.appendChild(ch);+  }+  (document.body || document.documentElement).appendChild(o);+}+function overlayClearFn() { var o = document.getElementById('__adom_finder_overlay__'); if (o) o.remove(); }++async function annotateShot(tabId, candidates, chosenIndex, x, y) {+  try {+    const boxes = candidates.map((c) => ({ rect: c.rect, occluded: !!c.occluded }));+    await chrome.scripting.executeScript({ target: { tabId }, func: overlayDrawFn, args: [boxes, chosenIndex == null ? -1 : chosenIndex, x == null ? null : x, y == null ? null : y] });+    const shot = await screenshot(tabId, { fullPage: false, maxWidth: 1568, resize: true });+    await chrome.scripting.executeScript({ target: { tabId }, func: overlayClearFn }).catch(() => {});+    return { base64: shot.base64, width: shot.width, height: shot.height };+  } catch (e) {+    try { await chrome.scripting.executeScript({ target: { tabId }, func: overlayClearFn }); } catch (_) {}+    return { __error: String((e && e.message) || e) };+  }+}++// Build the debug `finder` object (incl. shots + annotatedShot as base64; the bridge spills them to files).+async function buildFinder(tabId, query, candidates, chosenIndex, x, y, withShots) {+  const finder = { query, chosenIndex, candidates };+  if (withShots !== false) {+    try { const raw = await screenshot(tabId, { fullPage: false, maxWidth: 1568, resize: true }); if (raw && raw.base64) finder.shots = [{ label: 'viewport-raw', base64: raw.base64 }]; } catch (e) {}+    const ann = await annotateShot(tabId, candidates, chosenIndex, x, y);+    if (ann && ann.base64) finder.annotatedShot = { base64: ann.base64, width: ann.width, height: ann.height };+    else if (ann && ann.__error) finder.annotatedShotError = ann.__error;+  }+  return finder;+}++// Resolve target xy. When `rich`, also return the full candidate set + chosenIndex + structured query.+async function resolveCoords(tabId, args, rich) {   if (args.selector) {+    if (rich) {+      const cands = await evaluate(tabId, `(${selectorCandidatesFn.toString()})(${JSON.stringify(args.selector)})`);+      const list = Array.isArray(cands) ? cands : [];+      const ci = Math.max(0, list.findIndex((c) => c.chosen));+      if (!list.length) return { found: false, candidates: [], chosenIndex: -1, query: { selector: args.selector, intent: args.intent } };+      const el = list[ci] || list[0];+      return { found: true, x: el.x, y: el.y, info: { selector: args.selector, matchedCount: list.length, clickedRect: el.rect, clickedText: el.text }, candidates: list, chosenIndex: ci, query: { selector: args.selector, intent: args.intent } };+    }     const r = await evaluate(tabId, scoreExpr(args.selector, !!args.firstMatch));     if (!r || !r.found) return { found: false };     return { found: true, x: r.x, y: r.y, info: { selector: args.selector, matchedCount: r.count, clickedRect: r.rect, clickedText: r.text, modalDetected: r.modal } };   }   if (args.text) { // click/hover BY VISIBLE TEXT — the smart-click path-    const f = await find(tabId, args.text, 10);-    if (!f.ok || !f.elements.length) return { found: false };+    const f = await find(tabId, args.text, rich ? 16 : 10, rich);+    if (!f.ok || !f.elements.length) return { found: false, candidates: [], chosenIndex: -1, query: { text: args.text, role: args.role, intent: args.intent } };     const el = f.elements[0];-    return { found: true, x: el.x, y: el.y, info: { matchedText: el.text, matchedTag: el.tag, via: 'text', candidates: f.count } };+    const list = rich ? f.elements.map((c, i) => ({ index: i, ...c, chosen: i === 0 })) : null;+    return { found: true, x: el.x, y: el.y, info: { matchedText: el.text, matchedTag: el.tag, via: 'text', candidates: f.count }, candidates: list, chosenIndex: rich ? 0 : undefined, query: { text: args.text, role: args.role, intent: args.intent } };   }-  return { found: true, x: args.x, y: args.y, info: {} };+  return { found: true, x: args.x, y: args.y, info: {}, candidates: rich ? [] : undefined, chosenIndex: rich ? -1 : undefined, query: { intent: args.intent } }; }  // type ∈ move | hover | click | dblclick | rightclick | key | type@@ -266,23 +358,43 @@ export async function inputDispatch(tabId, args) {   await attach(tabId);   const type = (args.type || 'click').toLowerCase();   const showCursor = args.cursor === true;+  const resolveOnly = args.resolveOnly === true;          // dry-run: resolve + return finder, dispatch NOTHING+  const wantFinder = resolveOnly || args.debug === true;  // opt-in; default responses stay lean   try {-    if (type === 'key') { await pressKey(tabId, args.key); return { ok: true, dispatched: 'key', key: args.key }; }+    if (type === 'key') {+      if (resolveOnly) return { ok: true, resolveOnly: true, dispatched: false, key: args.key, _hint: 'key press has no finder target to resolve' };+      await pressKey(tabId, args.key); return { ok: true, dispatched: 'key', key: args.key };+    }     if (type === 'type') {+      let finder = null;+      if (wantFinder) {+        if (args.selector) { const c = await resolveCoords(tabId, args, true); finder = await buildFinder(tabId, c.query || { selector: args.selector }, c.candidates || [], c.chosenIndex == null ? -1 : c.chosenIndex, c.x, c.y); }+        else finder = { query: { intent: args.intent }, chosenIndex: -1, candidates: [], _note: 'typing into the focused field — no selector to resolve' };+      }+      if (resolveOnly) return { ok: true, resolveOnly: true, dispatched: false, ...(finder ? { finder } : {}) };       if (args.selector) {         const f = await evaluate(tabId, focusExpr(args.selector));-        if (!f || !f.found) return { ok: false, error: `selector not found: ${args.selector}`, _hint: 'check the selector or wait for the field' };+        if (!f || !f.found) return { ok: false, error: `selector not found: ${args.selector}`, _hint: 'check the selector or wait for the field', ...(finder ? { finder } : {}) };       }       await send(tabId, 'Input.insertText', { text: String(args.text || '') });-      return { ok: true, dispatched: 'type', length: String(args.text || '').length, via: args.selector ? 'selector' : 'focused' };+      return { ok: true, dispatched: 'type', length: String(args.text || '').length, via: args.selector ? 'selector' : 'focused', ...(finder ? { finder } : {}) };     }     // pointer types: move | hover | click | dblclick | rightclick-    const c = await resolveCoords(tabId, args);-    if (!c.found) return { ok: false, error: `selector not found: ${args.selector}`, _hint: 'element not visible/present; wait or refine the selector' };+    const c = await resolveCoords(tabId, args, wantFinder);+    if (!c.found) {+      const out = { ok: false, error: `target not found: ${args.selector || args.text || '(no selector/text)'}`, _hint: 'element not visible/present; wait or refine the selector/text' };+      if (wantFinder) out.finder = { query: c.query || {}, chosenIndex: -1, candidates: c.candidates || [] };+      return out;+    }     const { x, y, info } = c;+    // Build the debug finder (candidates + scores + annotated overlay shot) BEFORE acting, so a+    // resolveOnly dry-run and a debug click see the same pre-action page state.+    const finder = wantFinder ? await buildFinder(tabId, c.query || {}, c.candidates || [], c.chosenIndex == null ? 0 : c.chosenIndex, x, y) : null;+    const fin = finder ? { finder } : {};+    if (resolveOnly) return { ok: true, resolveOnly: true, dispatched: false, x, y, ...info, ...fin };     // The cursor label narrates intention (HD-style). args.intent overrides; else auto from element text.     const intent = args.intent || args.label;-    const tgt = (info.clickedText || '').trim().slice(0, 40);+    const tgt = (info.clickedText || info.matchedText || '').trim().slice(0, 40);     if (showCursor) { await cursor(tabId, 'move', x, y, intent || phaseLabel(type, tgt, 'find')); await sleep(args.cursorGlideMs || 450); }     if (type === 'move' || type === 'hover') {       if (showCursor) await cursor(tabId, 'move', x, y, intent || phaseLabel(type, tgt, 'act'));@@ -294,19 +406,19 @@ export async function inputDispatch(tabId, args) {       await sleep(40);       await send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }); // triggers :hover / mouseover / tooltips       await sleep(args.hoverDwellMs || 600);-      return { ok: true, dispatched: type, x, y, intentLabel: intent || phaseLabel(type, tgt, 'act'), ...info };+      return { ok: true, dispatched: type, x, y, intentLabel: intent || phaseLabel(type, tgt, 'act'), ...info, ...fin };     }     if (showCursor) await cursor(tabId, 'click', x, y, intent || phaseLabel(type, tgt, 'act')); // ripple + label     if (type === 'dblclick' || type === 'doubleclick') {       await mouseClick(tabId, x, y, 'left', 1); await mouseClick(tabId, x, y, 'left', 2);-      return { ok: true, dispatched: 'dblclick', x, y, ...info };+      return { ok: true, dispatched: 'dblclick', x, y, ...info, ...fin };     }     if (type === 'rightclick' || type === 'contextmenu') {       await mouseClick(tabId, x, y, 'right', 1);-      return { ok: true, dispatched: 'rightclick', x, y, ...info };+      return { ok: true, dispatched: 'rightclick', x, y, ...info, ...fin };     }     await mouseClick(tabId, x, y, args.button || 'left', args.clickCount || 1);-    return { ok: true, dispatched: 'click', via: args.selector ? 'selector' : 'coords', x, y, ...info };+    return { ok: true, dispatched: 'click', via: args.selector ? 'selector' : 'coords', x, y, ...info, ...fin };   } finally {     bumpIdle(tabId);   }