Pup - the AI's own browser

@@ -37,7 +37,7 @@ wait_done() { # poll apply-progress until done (max ~40s)   return 1 } -uia_buttons() { # prints name lines; also writes x|y|w|h|name rows to $SHOTDIR/rects.txt for the pixel gate+uia_buttons() { # $1 = rect file basename (default rects.txt); prints name lines; writes x|y|w|h|name rows   local PS='Add-Type -AssemblyName UIAutomationClient; $tray=[System.Windows.Automation.AutomationElement]::RootElement.FindFirst([System.Windows.Automation.TreeScope]::Children,(New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ClassNameProperty,"Shell_TrayWnd"))); $btns=$tray.FindAll([System.Windows.Automation.TreeScope]::Descendants,(New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ControlTypeProperty,[System.Windows.Automation.ControlType]::Button))); foreach($b in $btns){$r=$b.Current.BoundingRectangle; Write-Output ("{0}|{1}|{2}|{3}|{4}" -f [int]$r.X,[int]$r.Y,[int]$r.Width,[int]$r.Height,$b.Current.Name)}'   local B64; B64=$(python3 -c "import base64,sys;print(base64.b64encode(sys.stdin.read().encode('utf-16-le')).decode())" <<< "$PS")   timeout 90 $CLI shell_execute "{\"command\":\"powershell -NoProfile -EncodedCommand $B64\",\"reason\":\"mode-ralph: UIA taskbar button names + rects\"}" 2>&1 \@@ -45,7 +45,8 @@ uia_buttons() { # prints name lines; also writes x|y|w|h|name rows to $SHOTDIR/r import sys,json raw=sys.stdin.read();i=raw.find('{');d=json.loads(raw[i:]) if i>=0 else {} out=d.get('stdout') or ''-open('$SHOTDIR/rects.txt','w').write(out)+import os+open(os.path.join('$SHOTDIR', '${1:-rects.txt}'),'w').write(out) for l in out.splitlines():     p=l.split('|',4)     print(p[4] if len(p)==5 else l)"@@ -61,51 +62,116 @@ from PIL import Image, ImageDraw mode, label = sys.argv[1], sys.argv[2] import os SHOTDIR = os.environ.get("SHOTDIR","/tmp/mode-ralph-shots")-rows=[l.split('|',4) for l in open(f'{SHOTDIR}/rects.txt').read().splitlines() if l.count('|')>=4]-if mode=='plain':-    tgt=[r for r in rows if 'Chrome for Testing' in r[4]]-else:-    tgt=[r for r in rows if r[4].startswith('Adom Pup') or re.match(r'^(ti-crawl|mouser-run|digikey-run|wiki-in|wiki-out|dash-app|reclaimed|pup bridge)', r[4])]+def load(f):+    return [l.split('|',4) for l in open(f).read().splitlines() if l.count('|')>=4]+def pick(rows):+    if mode=='plain': return [r for r in rows if 'Chrome for Testing' in r[4]]+    return [r for r in rows if r[4].startswith('Adom Pup') or re.match(r'^(ti-crawl|mouser-run|digikey-run|wiki-in|wiki-out|dash-app|reclaimed|pup bridge)', r[4])]+rows=load(f'{SHOTDIR}/rects.txt')+tgt=pick(rows) if not tgt: print('PIXELGATE FAIL: no target button found'); sys.exit(1)-x,y,w,h = (int(tgt[0][i]) for i in range(4))-im=Image.open(f'{SHOTDIR}/{label}.png'); sc=im.width/2560.0-cx0,cx1=int(x*sc),int((x+w)*sc)-crop=im.crop((cx0,0,cx1,im.height)).convert('RGB')-px=list(crop.getdata()); teal=0-for r,g,b in px:-    if g>110 and b>100 and r<90 and abs(int(g)-int(b))<70: teal+=1-frac=teal/max(1,len(px))+# the taskbar REFLOWS for seconds after a mode switch (buttons materialize as stamps land) — a rect+# captured before the screenshot can point at a NEIGHBOR button by capture time (the RustDesk-swirl+# incident). rects2.txt is read AFTER the shot; if the target moved, the bar is still settling.+try:+    tgt2=pick(load(f'{SHOTDIR}/rects2.txt'))+    if not tgt2 or tgt2[0][4]!=tgt[0][4] or abs(int(tgt2[0][0])-int(tgt[0][0]))>4:+        print('PIXELGATE UNSTABLE: taskbar reflowed between rect read and screenshot'); sys.exit(2)+    tgt=tgt2+except FileNotFoundError:+    pass+# judge the MODE, not one unlucky button: a single straggler (the #53 hwnd-flaky class) can lag+# for tens of seconds while every sibling wears correct art — sample up to 4 targets, keep the best.+cands = tgt[:4] if mode!='plain' else tgt[:1]+import os.path+if not os.path.exists(f'{SHOTDIR}/{label}.png'):+    print('PIXELGATE FAIL: no fresh tray capture'); sys.exit(1)+im=Image.open(f'{SHOTDIR}/{label}.png')+import json as _json+def _map(x,w):+    try:+        cm=_json.load(open(f'{SHOTDIR}/{label}.json')).get('coordMap') or dict()+        i2s=cm.get('imageToScreen') or dict()+        sx=float(i2s.get('sx') or 0); ox=float(i2s.get('ox') or 0)+        if not sx: raise ValueError('no sx')+        a,b=int((x-ox)/sx),int((x+w-ox)/sx)+    except Exception:+        sc=im.width/2560.0; a,b=int(x*sc),int((x+w)*sc)+    a=max(0,min(im.width-2,a)); b=max(a+2,min(im.width,b))+    return a,b+best=None+for t in cands:+    x,y,w,h=(int(t[i]) for i in range(4))+    cx0,cx1=_map(x,w)+    crop=im.crop((cx0,0,cx1,im.height)).convert('RGB')+    W,H=crop.size+    px=crop.load(); teal=0; total=0+    for yy in range(H):+        for xx in range(W):+            if xx > W*0.45 and yy < H*0.62: continue+            r,g,b=px[xx,yy][:3]; total+=1+            # pup teal is #00b8b1: near-zero red, green~blue. CfT's own BLUE logo has cyan pixels+            # that a loose rule counted as teal (the 0.056-forever plain fail) — keep it tight.+            if g>120 and b>110 and r<70 and abs(int(g)-int(b))<45: teal+=1+    frac=teal/max(1,total)+    take = (best is None) or (frac > best[0] if mode != 'plain' else frac < best[0])+    if take:+        best=(frac,t,cx0,cx1)+frac,t,cx0,cx1=best ann=im.convert('RGB').copy(); d=ImageDraw.Draw(ann) d.rectangle([cx0-2,0,cx1+2,im.height-1],outline=(255,0,0),width=2) ann.resize((ann.width*2,ann.height*2),Image.LANCZOS).save(f'{SHOTDIR}/{label}-redbox.png')-crop.resize((crop.width*10,crop.height*10),Image.LANCZOS).save(f'{SHOTDIR}/{label}-btn10x.png')+im.crop((cx0,0,cx1,im.height)).resize(((cx1-cx0)*10,im.height*10),Image.LANCZOS).save(f'{SHOTDIR}/{label}-btn10x.png') ok = (frac < 0.02) if mode=='plain' else (frac > 0.04)-print(f"PIXELGATE {'PASS' if ok else 'FAIL'}: teal fraction {frac:.3f} on '{tgt[0][4][:50]}' ({'must be ~0 in plain' if mode=='plain' else 'pup art expected'})")+print(f"PIXELGATE {'PASS' if ok else 'FAIL'}: teal fraction {frac:.3f} on '{t[4][:50]}' ({'must be ~0 in plain' if mode=='plain' else 'best of ' + str(len(cands)) + ' pup buttons'})") sys.exit(0 if ok else 1) PYEOF } -shot_tray() { # shot_tray <label>-  local HW; HW=$(cat /tmp/tray_hwnd 2>/dev/null || echo 65952)+shot_tray() { # shot_tray <label> — MUST produce a FRESH capture; a stale file poisons every gate+  local HW; HW=$(cat /tmp/tray_hwnd 2>/dev/null || echo 0)+  if [ "$HW" = "0" ] || [ -z "$HW" ]; then HW=$(resolve_tray_hwnd); fi   timeout 60 $CLI desktop_screenshot_window "{\"hwnd\":$HW}" >/dev/null 2>&1-  local P; P=$(ls -t ~/project/screenshots/adom-bridge/window/*-${HW}*.full.png 2>/dev/null | head -1)-  [ -n "$P" ] && cp "$P" "$SHOTDIR/$1.png"+  local P; P=$(find ~/project/screenshots/adom-bridge/window -name "*-${HW}.full.png" -newermt "-45 seconds" 2>/dev/null | head -1)+  if [ -z "$P" ]; then+    # capture failed or hwnd stale (Explorer restart) — re-resolve once and retry+    HW=$(resolve_tray_hwnd); echo "$HW" > /tmp/tray_hwnd+    timeout 60 $CLI desktop_screenshot_window "{\"hwnd\":$HW}" >/dev/null 2>&1+    P=$(find ~/project/screenshots/adom-bridge/window -name "*-${HW}.full.png" -newermt "-45 seconds" 2>/dev/null | head -1)+  fi+  if [ -z "$P" ]; then echo "  SHOT_TRAY FAILED: no fresh capture (hwnd $HW) — pixel gate will be skipped as FAIL" >&2; rm -f "$SHOTDIR/$1.png"; return 1; fi+  cp "$P" "$SHOTDIR/$1.png"+}++resolve_tray_hwnd() {+  local PS='Add-Type -AssemblyName UIAutomationClient; $tray=[System.Windows.Automation.AutomationElement]::RootElement.FindFirst([System.Windows.Automation.TreeScope]::Children,(New-Object System.Windows.Automation.PropertyCondition([System.Windows.Automation.AutomationElement]::ClassNameProperty,"Shell_TrayWnd"))); $tray.Current.NativeWindowHandle'+  local B64; B64=$(python3 -c "import base64,sys;print(base64.b64encode(sys.stdin.read().encode('utf-16-le')).decode())" <<< "$PS")+  timeout 60 $CLI shell_execute "{\"command\":\"powershell -NoProfile -EncodedCommand $B64\",\"reason\":\"mode-ralph: resolve tray hwnd\"}" 2>&1 \+    | python3 -c "import sys,json;raw=sys.stdin.read();i=raw.find('{');d=json.loads(raw[i:]) if i>=0 else {};print((d.get('stdout') or '0').strip())" } -verify_state() { # verify_state <mode> <label>+verify_state() { # verify_state <mode> <label> — retries while the bar is still reflowing   local mode="$1" label="$2"-  local B; B=$(uia_buttons)+  local B PG PGOK attempt   local N_PUP N_THREAD N_CFT-  N_PUP=$(echo "$B" | grep -c "Adom Pup" || true)-  N_THREAD=$(echo "$B" | grep -Ec "^(ti-crawl|mouser-run|digikey-run|wiki-in|wiki-out|dash-app|reclaimed|pup bridge maintenance|pup-dashboard)" || true)-  N_CFT=$(echo "$B" | grep -c "Chrome for Testing" || true)-  shot_tray "$label"-  local PG; PG=$(SHOTDIR="$SHOTDIR" pixel_gate "$mode" "$label"); local PGOK=$?+  for attempt in 1 2 3 4 5; do+    B=$(uia_buttons rects.txt)+    N_PUP=$(echo "$B" | grep -c "Adom Pup" || true)+    N_THREAD=$(echo "$B" | grep -Ec "^(ti-crawl|mouser-run|digikey-run|wiki-in|wiki-out|dash-app|reclaimed|pup bridge maintenance|pup-dashboard)" || true)+    N_CFT=$(echo "$B" | grep -c "Chrome for Testing" || true)+    shot_tray "$label"+    uia_buttons rects2.txt >/dev/null+    PG=$(SHOTDIR="$SHOTDIR" pixel_gate "$mode" "$label"); PGOK=$?+    [ $PGOK -eq 0 ] && break+    # nonzero: bar reflowing (2) or a transient mid-rebuild frame measured wrong (1) — settle + retry+    echo "  (gate attempt $attempt: $PG — settling 6s)"+    sleep 6+  done   echo "  $PG"   case "$mode" in     plain)-      if [ "$N_PUP" -eq 0 ] && [ "$N_THREAD" -eq 0 ] && [ $PGOK -eq 0 ]; then PASS=$((PASS+1)); echo "  PASS [$label] plain: 0 branded buttons (CfT groups: $N_CFT) + pixel gate"; return 0-      else FAIL=$((FAIL+1)); echo "  FAIL [$label] plain: $N_PUP 'Adom Pup' + $N_THREAD thread buttons still on the bar"; echo "$B" | grep -E "Adom Pup|^(ti-crawl|mouser-run|digikey-run|wiki-in|wiki-out|dash-app|reclaimed|pup bridge)" | sed 's/^/    /'; return 1; fi ;;+      if [ "$N_PUP" -eq 0 ] && [ "$N_THREAD" -eq 0 ] && [ "$N_CFT" -eq 1 ] && [ $PGOK -eq 0 ]; then PASS=$((PASS+1)); echo "  PASS [$label] plain: 0 branded buttons, exactly ONE CfT group + pixel gate"; return 0+      else FAIL=$((FAIL+1)); echo "  FAIL [$label] plain: $N_PUP 'Adom Pup' + $N_THREAD thread buttons, $N_CFT CfT groups (must be exactly 1)"; echo "$B" | grep -E "Adom Pup|^(ti-crawl|mouser-run|digikey-run|wiki-in|wiki-out|dash-app|reclaimed|pup bridge)" | sed 's/^/    /'; return 1; fi ;;     aumid)       if [ "$N_PUP" -ge 5 ] && [ $PGOK -eq 0 ]; then PASS=$((PASS+1)); echo "  PASS [$label] aumid: $N_PUP 'Adom Pup' per-window buttons + pixel gate"; return 0       else FAIL=$((FAIL+1)); echo "  FAIL [$label] aumid: only $N_PUP 'Adom Pup' buttons"; return 1; fi ;;@@ -123,11 +189,13 @@ round() { # round <mode> <ov> <label> — clean switch, wait, verify   verify_state "$1" "$3" } -rapid_round() { # rapid_round <modeA> <modeB> <label> — the race: click A, then B ~1.2s later; B must win+rapid_round() { # rapid_round <modeA> <modeB> <label> — the race: click A, then B ~1.2s later; B must win.+  # B lands with overlay OFF: the group badge favicon is racey (most-recent window wins), so a teal+  # badge would poison the base-art teal measurement nondeterministically.   echo "== RAPID $3: $1 then $2 after 1.2s (the in-flight-stamp race) =="   switch_state "$1" "on"   sleep 1.2-  switch_state "$2" "on"+  switch_state "$2" "off"   wait_done || echo "  (apply-progress never reported done — verifying anyway)"   sleep 5   verify_state "$2" "$3"@@ -135,12 +203,15 @@ rapid_round() { # rapid_round <modeA> <modeB> <label> — the race: click A, the  round tiles on  r1-tiles round aumid on  r2-aumid-round plain on  r3-plain+round plain off r3-plain rapid_round aumid plain r4-rapid-aumid-then-plain round tiles on  r5-tiles rapid_round tiles plain r6-rapid-tiles-then-plain round aumid on  r7-aumid-round "$FINAL_MODE" "$FINAL_OV" r8-final-$FINAL_MODE+round "$FINAL_MODE" off r8-final-$FINAL_MODE+# restore the user's chosen overlay setting (measurement used overlay-off for determinism)+switch_state "$FINAL_MODE" "$FINAL_OV"+echo "== restored final state: $FINAL_MODE + overlay $FINAL_OV =="  echo echo "== MODE-RALPH RESULT: $PASS passed, $FAIL failed (shots in $SHOTDIR) =="