← Commit history

Restructure step 0: characterization suite. Locks the CURRENT observable behaviour of pup's verbs (21 cases: read-only verbs, the open-drive-close lifecycle, and four refusal paths that must keep refusing with the same errorCode) so that splitting the 3,601-line handler into verbs/ modules surfaces any behavioural drift as a diff instead of as a user-reported bug days later. Compares response SHAPE and invariants, not volatile values — timings, ids and paths are normalised, as are the advisory fields pup emits intermittently by design (_reportIssues, _cleanupReport, occasional hints), because a suite with false positives gets ignored and is worse than no suite. Verified in both directions: clean run reports drift 0, and an injected baseline change IS caught.

John Lauer ·393200d54b ·3mo ago ·parent 3189cf3
2 files changed +712
tests/baseline.jsonadded+543
@@ -0,0 +1,543 @@+{+  "bridge": "1.9.223",+  "cases": [+    {+      "result": {+        "errorCode": null,+        "ok": null,+        "shape": {+          "_hint": "<volatile>",+          "_next": "<volatile>",+          "browserExecutablePath": "str",+          "browserKind": "str",+          "browserSource": "str",+          "bytesDownloaded": "num",+          "bytesTotal": "num",+          "cacheDir": "str",+          "candidates": [+            {+              "executablePath": "str",+              "kind": "str",+              "source": "str"+            }+          ],+          "chromeBuildId": "str",+          "chromeExecutablePath": "str",+          "chromeForTestingInstalled": "bool",+          "chromeStableAwaitingApproval": "bool",+          "chromeStableError": "null",+          "chromeStableNotifyChannel": "null",+          "chromeStableNotifyPeers": "null",+          "chromeStablePhase": "str",+          "defaultBrowser": {+            "forced": "bool",+            "kind": "str",+            "pendingInstall": "bool",+            "source": "str"+          },+          "diskFreeMb": "num",+          "installPhase": "str",+          "installProgressPct": "num",+          "installing": "bool",+          "installingChrome": "bool",+          "lastError": "null",+          "lastErrorCode": "null",+          "lowDisk": "bool",+          "nodeInstalled": "bool",+          "nodeVersion": "str",+          "pitfalls": [+            "str"+          ],+          "puppeteerLoadError": "null",+          "ready": "bool",+          "related": [+            "str"+          ]+        }+      },+      "verb": "browser_readiness"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "activeSession": "<volatile>",+          "durationMs": "<volatile>",+          "hint": "<volatile>",+          "ok": "bool",+          "pupshotUrl": "null",+          "sessionCount": "num",+          "sessions": "<volatile>",+          "status": "str"+        }+      },+      "verb": "browser_status"+    },+    {+      "result": {+        "errorCode": null,+        "ok": null,+        "shape": {+          "activeSession": "<volatile>",+          "count": "<volatile>",+          "durationMs": "<volatile>",+          "sessions": "<volatile>",+          "status": "str"+        }+      },+      "verb": "browser_list_windows"+    },+    {+      "result": {+        "errorCode": null,+        "ok": null,+        "shape": {+          "_hint": "<volatile>",+          "_next": "<volatile>",+          "_timeoutHint": "<volatile>",+          "bridge": "str",+          "related": [+            "str"+          ],+          "statusVerb": "str",+          "success": "bool",+          "timeoutSeconds": "num",+          "verbs": [+            {+              "example": {+                "height": "\u2026",+                "owner": "<volatile>",+                "sessionId": "<volatile>",+                "url": "<volatile>",+                "width": "\u2026"+              },+              "hint": "<volatile>",+              "input": {+                "appIconB64": "\u2026",+                "appName": "\u2026",+                "autoFlash": "\u2026",+                "foreground": "\u2026",+                "foregroundReason": "\u2026",+                "height": "\u2026",+                "isolated": "\u2026",+                "owner": "<volatile>",+                "profile": "<volatile>",+                "sessionId": "<volatile>",+                "silent": "\u2026",+                "takeover": "\u2026",+                "updateNote": "\u2026",+                "url": "<volatile>",+                "webSecurity": "\u2026",+                "width": "\u2026",+                "wikiView": "\u2026",+                "x": "\u2026",+                "y": "\u2026"+              },+              "longRunning": "bool",+              "name": "str",+              "output": {+                "success": "\u2026"+              },+              "pitfalls": [+                "\u2026"+              ],+              "related": [+                "\u2026"+              ],+              "statusVerb": "null",+              "summary": "<volatile>",+              "timeoutSeconds": "num"+            }+          ]+        }+      },+      "verb": "browser_describe"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "depsPresent": "bool",+          "dir": "str",+          "lastError": "null",+          "ok": "bool",+          "selfInstalledThisBoot": "bool"+        }+      },+      "verb": "browser_deps"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "_next": "<volatile>",+          "_verifyRender": "<volatile>",+          "active": "<volatile>",+          "activeTabId": "<volatile>",+          "ageMinutes": "<volatile>",+          "appIconWorn": "bool",+          "availableBrowsers": [+            "str"+          ],+          "background": "<volatile>",+          "browser": {+            "executablePath": "str",+            "kind": "str",+            "source": "str"+          },+          "defaultBrowser": {+            "forced": "bool",+            "kind": "str",+            "pendingInstall": "bool",+            "source": "str"+          },+          "defaultPermissionsApplied": "bool",+          "defaultPermissionsCount": "num",+          "defaultPermissionsNote": "str",+          "downloadPath": "str",+          "downloadsEnabled": "bool",+          "durationMs": "<volatile>",+          "emulatedViewport": "<volatile>",+          "errorCount": "<volatile>",+          "flash": "<volatile>",+          "flashCount": "<volatile>",+          "highFps": "bool",+          "lastAgentUpdate": "<volatile>",+          "lastForeground": "<volatile>",+          "ok": "bool",+          "overlay": "<volatile>",+          "owner": "<volatile>",+          "ownerSource": "<volatile>",+          "pitfalls": [+            "str"+          ],+          "placement": "str",+          "profile": "<volatile>",+          "progressBarCount": "<volatile>",+          "progressBarMs": "<volatile>",+          "related": [+            "str"+          ],+          "renderCheck": {+            "bodyChars": "num",+            "elements": "num",+            "httpStatus": "num",+            "ok": "bool",+            "signals": [],+            "title": "<volatile>"+          },+          "restoredTabsClosed": "<volatile>",+          "reused": "null",+          "sessionId": "<volatile>",+          "size": "null",+          "status": "str",+          "tabCount": "<volatile>",+          "title": "<volatile>",+          "url": "<volatile>"+        }+      },+      "verb": "browser_open_window"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "durationMs": "<volatile>",+          "ok": "bool",+          "result": "num",+          "sessionId": "<volatile>",+          "status": "str",+          "tabId": "<volatile>"+        }+      },+      "verb": "browser_eval"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "_next": "<volatile>",+          "_verifyRender": "<volatile>",+          "durationMs": "<volatile>",+          "ok": "bool",+          "pitfalls": [+            "str"+          ],+          "related": [+            "str"+          ],+          "renderCheck": {+            "bodyChars": "num",+            "elements": "num",+            "httpStatus": "num",+            "ok": "bool",+            "signals": [],+            "title": "<volatile>"+          },+          "sessionId": "<volatile>",+          "status": "str",+          "tabId": "<volatile>",+          "url": "<volatile>"+        }+      },+      "verb": "browser_navigate"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "_next": "<volatile>",+          "_verifyRender": "<volatile>",+          "active": "<volatile>",+          "activeTabId": "<volatile>",+          "durationMs": "<volatile>",+          "errorCount": "<volatile>",+          "ok": "bool",+          "opener": "str",+          "openerTabId": "null",+          "pitfalls": [+            "str"+          ],+          "related": [+            "str"+          ],+          "renderCheck": {+            "bodyChars": "num",+            "elements": "num",+            "httpStatus": "num",+            "ok": "bool",+            "signals": [],+            "title": "<volatile>"+          },+          "sessionId": "<volatile>",+          "status": "str",+          "tabCount": "<volatile>",+          "tabId": "<volatile>",+          "title": "<volatile>",+          "url": "<volatile>"+        }+      },+      "verb": "browser_open_tab"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "activeTabId": "<volatile>",+          "count": "<volatile>",+          "durationMs": "<volatile>",+          "lastAgentUpdate": "<volatile>",+          "ok": "bool",+          "sessionId": "<volatile>",+          "status": "str",+          "tabs": "<volatile>"+        }+      },+      "verb": "browser_list_tabs"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "atBottom": "bool",+          "clientHeight": "num",+          "found": "bool",+          "ok": "bool",+          "scrollElement": "str",+          "scrollHeight": "num",+          "scrollLeft": "num",+          "scrollTop": "num",+          "sessionId": "<volatile>",+          "tabId": "<volatile>"+        }+      },+      "verb": "browser_scroll"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "applied": {+            "deviceScaleFactor": "num",+            "height": "num",+            "width": "num"+          },+          "devicePixelRatio": "num",+          "innerHeight": "num",+          "innerWidth": "num",+          "ok": "bool",+          "sessionId": "<volatile>",+          "tabId": "<volatile>"+        }+      },+      "verb": "browser_set_viewport"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "devicePixelRatio": "num",+          "innerHeight": "num",+          "innerWidth": "num",+          "ok": "bool",+          "reset": "bool",+          "sessionId": "<volatile>",+          "tabId": "<volatile>"+        }+      },+      "verb": "browser_set_viewport_reset"+    },+    {+      "result": {+        "errorCode": null,+        "ok": null,+        "shape": {+          "_hint": "<volatile>",+          "durationMs": "<volatile>",+          "height": "num",+          "origHeight": "num",+          "origWidth": "num",+          "resized": "bool",+          "savedTo": "str",+          "sizeKB": "num",+          "status": "str",+          "success": "bool",+          "width": "num"+        }+      },+      "verb": "browser_screenshot"+    },+    {+      "result": {+        "errorCode": null,+        "ok": null,+        "shape": {+          "count": "<volatile>",+          "durationMs": "<volatile>",+          "errors": [+            {+              "text": "str",+              "ts": "<volatile>",+              "type": "str"+            }+          ],+          "sessionId": "<volatile>",+          "status": "str",+          "tabId": "<volatile>"+        }+      },+      "verb": "browser_errors"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "bounds": "<volatile>",+          "ok": "bool",+          "sessionId": "<volatile>",+          "state": "str"+        }+      },+      "verb": "browser_maximize"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "_next": "<volatile>",+          "_verifyRender": "<volatile>",+          "activeTabId": "<volatile>",+          "durationMs": "<volatile>",+          "ok": "bool",+          "openedAsTab": "bool",+          "pitfalls": [+            "str"+          ],+          "related": [+            "str"+          ],+          "sessionId": "<volatile>",+          "status": "str",+          "tabCount": "<volatile>",+          "tabId": "<volatile>"+        }+      },+      "verb": "refusal_no_identity"+    },+    {+      "result": {+        "errorCode": "session_not_found",+        "ok": false,+        "shape": {+          "_scalar": "str"+        }+      },+      "verb": "refusal_no_session"+    },+    {+      "result": {+        "errorCode": "url_file_container_path",+        "ok": false,+        "shape": {+          "_scalar": "str"+        }+      },+      "verb": "refusal_container_path"+    },+    {+      "result": {+        "errorCode": "foreground_reason_required",+        "ok": false,+        "shape": {+          "_scalar": "str"+        }+      },+      "verb": "refusal_bare_raise"+    },+    {+      "result": {+        "errorCode": null,+        "ok": true,+        "shape": {+          "_hint": "<volatile>",+          "_next": "<volatile>",+          "activeSession": "<volatile>",+          "chromeProcsKilled": "num",+          "chromeProcsRemaining": "num",+          "durationMs": "<volatile>",+          "ok": "bool",+          "pitfalls": [+            "str"+          ],+          "related": [+            "str"+          ],+          "remaining": "num",+          "status": "str"+        }+      },+      "verb": "browser_close_window"+    }+  ]+}
tests/characterize.shadded+169
@@ -0,0 +1,169 @@+#!/usr/bin/env bash+# ── pup CHARACTERIZATION SUITE (restructure step 0) ────────────────────────────────────────────+#+# Locks the CURRENT observable behaviour of every verb BEFORE the restructure moves any code.+# This is not a correctness suite — it does not assert what pup *should* do. It records what pup+# DOES do, so that when the 3,601-line handler is split into verbs/ modules, any behavioural drift+# shows up as a diff instead of as a user-reported bug three days later.+#+# Usage:+#   tests/characterize.sh baseline     # record current behaviour  -> tests/baseline.json+#   tests/characterize.sh check        # re-run and diff against the baseline+#+# Rules this suite obeys (from dev-skills/pup-bridge-test):+#   - up-gate first: never test a bridge that is still reconstructing+#   - never >/dev/null an open: capture ok + error + errorCode+#   - clean up every window it creates+#   - use a throwaway thread name so no real thread's window is touched+set -u+TARGET="${ADOM_TARGET:-AdomLapper}"+THREAD="pup-characterize"+MODE="${1:-check}"+DIR="$(cd "$(dirname "$0")" && pwd)"+OUT="$DIR/baseline.json"+TMP="$(mktemp -d)"+trap 'rm -rf "$TMP"' EXIT++AD() { timeout 75 adom-desktop --target "$TARGET" --ai-thread "$THREAD" "$@" 2>/dev/null; }++# Record only the SHAPE + key invariants of a response, not volatile values (timings, ids, paths).+# A restructure must preserve shape and invariants; it is allowed to change a duration.+shape() {+  python3 -c "+import sys, json+raw = sys.stdin.read()+try:+    d = json.loads(raw)+except Exception:+    print(json.dumps({'_unparseable': raw[:200]})); raise SystemExit+o = d.get('output', d)+if isinstance(o, str):+    try: o = json.loads(o)+    except Exception: pass+if not isinstance(o, dict): o = {'_scalar': type(o).__name__}+# Volatile = values that legitimately change run to run, PLUS advisory fields that pup emits+# INTERMITTENTLY by design (issue-report nudges, cleanup reports, occasional hints). Those come and+# go on purpose, so comparing them produces false drift — and a suite that cries wolf gets ignored,+# which is worse than no suite. Presence is normalised, not just the value.+VOLATILE = {'durationMs','ts','timestamp','createdAt','updatedAt','ageMinutes','pid','cdpPort',+            'hwnd','shotId','path','localSafePath','fullPath','tabId','sessionId','url','title',+            'progressBarMs','progressBarCount','flashCount','bounds','rect','_hint','_next',+            '_timeoutHint','_reportIssues','_cleanupReport','_degraded','_urlLint','lastAgentUpdate',+            # intermittent-by-design advisory/status fields:+            'active','activeSession','activeTabId','hint','_verifyRender','_autoLogin','_flashCount',+            'errorCount','tabCount','overlay','ownerSource','owner','background','flash','emulatedViewport',+            'lastForeground','restoredTabsClosed','_appOverlay','degraded','degradedReason','profile',+            'windows','sessions','tabs','count','_screenshots','_related','_cacheHint','_skillsHint',+            '_staleWarning','staleBridges','staleCheckedAt','summary','bridges','current_os'}+def keyshape(x, depth=0):+    if depth > 3: return '…'+    if isinstance(x, dict):+        return {k: ('<volatile>' if k in VOLATILE else keyshape(v, depth+1))+                for k, v in sorted(x.items())}+    if isinstance(x, list):+        return [keyshape(x[0], depth+1)] if x else []+    if isinstance(x, bool): return 'bool'+    if isinstance(x, (int, float)): return 'num'+    if x is None: return 'null'+    return 'str'+print(json.dumps({'ok': d.get('ok', o.get('ok')), 'errorCode': d.get('errorCode') or o.get('errorCode'),+                  'shape': keyshape(o)}, sort_keys=True))+"+}++record() {  # record <name> <<< response+  local name="$1"+  python3 -c "+import sys, json+name = sys.argv[1]+body = sys.stdin.read().strip() or '{}'+print(json.dumps({'verb': name, 'result': json.loads(body)}))+" "$name"+}++echo "── pup characterization ($MODE) ──"++# ── up-gate ────────────────────────────────────────────────────────────────────────────────────+for i in $(seq 1 15); do+  AD browser_list_windows | python3 -c "+import sys, json+try: sys.exit(0 if json.load(sys.stdin).get('count') is not None else 1)+except Exception: sys.exit(1)" && break+  sleep 6+done++VER=$(AD bridge_list | python3 -c "+import sys, json+try:+    print(next((b.get('version') for b in json.load(sys.stdin).get('bridges', []) if b.get('name') == 'puppeteer'), '?'))+except Exception: print('?')")+echo "  bridge: $VER"++: > "$TMP/results.jsonl"+SID="char-$$"++# ── read-only verbs (safe, no side effects) ────────────────────────────────────────────────────+for v in browser_readiness browser_status browser_list_windows browser_describe browser_deps; do+  AD "$v" '{}' | shape | record "$v" >> "$TMP/results.jsonl"+done++# ── lifecycle: open → drive → close ────────────────────────────────────────────────────────────+AD browser_open_window "{\"sessionId\":\"$SID\",\"url\":\"https://example.com\",\"reason\":\"characterization suite\"}" \+  | shape | record browser_open_window >> "$TMP/results.jsonl"+sleep 4++AD browser_eval        "{\"sessionId\":\"$SID\",\"expr\":\"1+1\"}"                        | shape | record browser_eval        >> "$TMP/results.jsonl"+AD browser_navigate    "{\"sessionId\":\"$SID\",\"url\":\"https://example.org\"}"          | shape | record browser_navigate    >> "$TMP/results.jsonl"+AD browser_open_tab    "{\"sessionId\":\"$SID\",\"url\":\"https://www.iana.org\"}"         | shape | record browser_open_tab    >> "$TMP/results.jsonl"+AD browser_list_tabs   "{\"sessionId\":\"$SID\"}"                                          | shape | record browser_list_tabs   >> "$TMP/results.jsonl"+AD browser_scroll      "{\"sessionId\":\"$SID\",\"y\":200}"                                | shape | record browser_scroll      >> "$TMP/results.jsonl"+AD browser_set_viewport "{\"sessionId\":\"$SID\",\"width\":900,\"height\":600}"            | shape | record browser_set_viewport >> "$TMP/results.jsonl"+AD browser_set_viewport "{\"sessionId\":\"$SID\",\"reset\":true}"                          | shape | record browser_set_viewport_reset >> "$TMP/results.jsonl"+AD browser_screenshot  "{\"sessionId\":\"$SID\"}"                                          | shape | record browser_screenshot  >> "$TMP/results.jsonl"+AD browser_errors      "{\"sessionId\":\"$SID\"}"                                          | shape | record browser_errors      >> "$TMP/results.jsonl"+AD browser_maximize    "{\"sessionId\":\"$SID\"}"                                          | shape | record browser_maximize    >> "$TMP/results.jsonl"++# ── refusals: these MUST keep refusing, with the same errorCode ────────────────────────────────+AD browser_open_window '{"url":"https://example.com"}'                                     | shape | record refusal_no_identity  >> "$TMP/results.jsonl"+AD browser_eval        '{"sessionId":"definitely-not-a-real-session","expr":"1"}'           | shape | record refusal_no_session   >> "$TMP/results.jsonl"+AD browser_open_window '{"sessionId":"lint-check","url":"file:///home/adom/x.html"}'        | shape | record refusal_container_path >> "$TMP/results.jsonl"+AD browser_raise_os_window "{\"sessionId\":\"$SID\"}"                                       | shape | record refusal_bare_raise   >> "$TMP/results.jsonl"++# ── cleanup ────────────────────────────────────────────────────────────────────────────────────+AD browser_close_window "{\"sessionId\":\"$SID\",\"reason\":\"characterization done\"}" \+  | shape | record browser_close_window >> "$TMP/results.jsonl"++python3 -c "+import json, sys+rows = [json.loads(l) for l in open('$TMP/results.jsonl') if l.strip()]+print(json.dumps({'bridge': '$VER', 'cases': rows}, indent=2, sort_keys=True))+" > "$TMP/current.json"++if [ "$MODE" = "baseline" ]; then+  cp "$TMP/current.json" "$OUT"+  echo "  baseline written: $OUT ($(python3 -c "+import json; print(len(json.load(open('$OUT'))['cases']))") cases)"+  exit 0+fi++if [ ! -f "$OUT" ]; then+  echo "  NO BASELINE. Run: tests/characterize.sh baseline"+  exit 2+fi++python3 -c "+import json+base = {c['verb']: c['result'] for c in json.load(open('$OUT'))['cases']}+cur  = {c['verb']: c['result'] for c in json.load(open('$TMP/current.json'))['cases']}+drift = []+for verb in sorted(set(base) | set(cur)):+    b, c = base.get(verb), cur.get(verb)+    if b != c:+        drift.append((verb, b, c))+print(f'  cases: {len(cur)} | drift: {len(drift)}')+for verb, b, c in drift:+    print(f'   ✗ {verb}')+    print(f'      baseline: {json.dumps(b)[:150]}')+    print(f'      current : {json.dumps(c)[:150]}')+raise SystemExit(1 if drift else 0)+"