AI Flow

#!/usr/bin/env python3
"""One sub-readme per finished aiflow step on the board's project page, and the main README's
"How this board was built" section (one intro per step, linking to its sub-readme) kept current.

  python3 project-steps.py <run dir> <owner/slug> [<step> ...]     (no step: every step visited so far)

With a page folder (`project-page --page-dir <folder>`, run.json projectPageDir, issue #85) the step
pages are <folder>/steps/<nn>-<step>/ and the section goes in <folder>/README.md.

steps/<nn>-<step>/README.md holds the step's evidence: what it does, each visit (and why a return
came back to it), the proof the binary recorded, the screenshots and the AI's drawings, and the
Antigravity tokens the step used. The main README section sits between
<!-- aiflow:steps --> and <!-- /aiflow:steps -->; everything else on the page is left as it is."""
import json, os, re, shutil, subprocess, sys

BEGIN, END = "<!-- aiflow:steps -->", "<!-- /aiflow:steps -->"


def clip(text, n=600):
    """Error text within n characters, keeping its beginning (the cause) and its end (issue #76)."""
    text = text or ""
    if len(text) <= n:
        return text
    head = int(n * 0.7)
    return text[:head] + " ... " + text[len(text) - (n - head - 5):]


def load(run):
    data = json.load(open(os.path.join(run, "run.json")))
    events = []
    for line in open(os.path.join(run, "run.jsonl")) if os.path.isfile(os.path.join(run, "run.jsonl")) else []:
        try: events.append(json.loads(line))
        except Exception: pass
    return data, events


def flow_of(data):
    return data.get("flow") or {}


def steps_done(data):
    seq = data.get("sequence") or []
    seen = [s for s in seq if (data.get("steps", {}).get(s) or {}).get("visits")]
    return seq, seen


def tokens_by_step(data):
    snaps = [s for s in data.get("usage") or [] if isinstance(s.get("agyTokens"), dict)]
    out = {}
    for a, b in zip(snaps, snaps[1:]):
        st = (a.get("when") or "").replace("step:", "")
        d = {k: max(0, (b["agyTokens"].get(k) or 0) - (a["agyTokens"].get(k) or 0)) for k in ("calls", "freshIn", "cachedIn", "out")}
        if not d["calls"]: continue
        o = out.setdefault(st, {k: 0 for k in d})
        for k in d: o[k] += d[k]
    return out


def brief(v, depth=0):
    """A short human reading of an outcome: scalars and short lists, nested once."""
    if isinstance(v, dict):
        rows = []
        for k, x in v.items():
            if isinstance(x, (str, int, float, bool)) and len(str(x)) < 160: rows.append(f"- **{k}**: {x}")
            elif isinstance(x, list) and x and len(x) <= 8 and all(isinstance(i, (str, int, float)) for i in x): rows.append(f"- **{k}**: {', '.join(map(str, x))}")
            elif isinstance(x, list): rows.append(f"- **{k}**: {len(x)} item(s)")
            elif isinstance(x, dict) and depth == 0: rows.append(f"- **{k}**: " + "; ".join(f"{a} {b}" for a, b in list(x.items())[:8] if isinstance(b, (str, int, float, bool))))
        return "\n".join(rows)
    return str(v)


def page_dir(data, page):
    """The project page's folder (projectPageDir) when `page` is the run's project page, else None."""
    d = data.get("projectPageDir") if data.get("projectPage") == page else None
    return d.strip("/") if isinstance(d, str) and d.strip("/") else None


def under(d, rel):
    return f"{d}/{rel}" if d else rel


def clips_target(data):
    """The page and folder the clips were pushed to: clips --page, else the run's page (as the binary does)."""
    if data.get("clipsPage"):
        return data["clipsPage"], data.get("clipsPageDir")
    if data.get("page"):
        return data["page"], data.get("pageDir")
    return data.get("projectPage"), page_dir(data, data.get("projectPage"))


def sub_readme(run, data, events, step, n, stage, folder=None):
    flow = flow_of(data)
    sdef = next((s for s in flow.get("steps", []) if s.get("name") == step), {})
    title = (flow.get("titles") or {}).get(step, step)
    does = sdef.get("does") or (flow.get("short") or {}).get(step, "")
    visits = (data.get("steps", {}).get(step) or {}).get("visits") or []
    d = os.path.join(stage, under(folder, "steps"), f"{n:02d}-{step}"); os.makedirs(d, exist_ok=True)
    md = [f"# Step {n}: {title}", "", does, ""]
    md += ["## Visits", ""] + [f"- {v.get('at', '')}" + (f" (a return: {v.get('why')})" if v.get("back") else "") for v in visits] + [""]
    proofs = (flow.get("proofs") or {}).get(step) or []
    outs = data.get("outcomes") or {}
    shown = [k for k in proofs if k in outs] + [k for k in outs if k.startswith(step) and k not in proofs]
    if shown:
        md += ["## What the binary checked", ""]
        for k in shown: md += [f"### {k}", "", brief(outs[k]), ""]
    ev = [e for e in data.get("evidence") or [] if e.get("step") == step]
    if ev:
        md += ["## Evidence", ""] + [f"- {e.get('kind')}: `{os.path.basename(e.get('file', ''))}`" + (" (pass)" if e.get("pass") else " (fail)" if e.get("pass") is False else "") for e in ev] + [""]
    pics = []
    for e in events:
        if e.get("event") == "artifact" and e.get("step") == step and str(e.get("file", "")).lower().endswith((".png", ".jpg", ".jpeg", ".webp")):
            f = e["file"] if os.path.isabs(e["file"]) else os.path.join(run, e["file"])
            if os.path.isfile(f) and f not in [p[0] for p in pics]: pics.append((f, e.get("caption") or e.get("when") or e.get("kind") or ""))
    for f in sorted(os.listdir(run)):
        if f.startswith(f"shot-{step}-") and f.endswith(".png") and os.path.join(run, f) not in [p[0] for p in pics]:
            pics.append((os.path.join(run, f), f.replace(".png", "").replace(f"shot-{step}-", "visit ")))
    if pics:
        md += ["## Screenshots and drawings", ""]
        for f, cap in pics[:16]:
            name = os.path.basename(f); shutil.copy(f, os.path.join(d, name))
            md += [f"**{cap}**", "", f"![{cap}]({name})", ""]
    # the step's clips (pushed by the clips page as <folder>/docs/clips/<stem>-10x.mp4), in the wiki's video widget
    page, cdir = clips_target(data)
    vids = []
    for c in data.get("captures") or []:
        if c.get("step") != step or c.get("discarded") or not c.get("file"): continue
        stem = os.path.splitext(os.path.basename(c["file"]))[0]
        if os.path.isfile(os.path.join(run, f"{stem}-10x.mp4")): vids.append((stem, c.get("visit")))
    if vids and page:
        md += ["## Clips", ""]
        for stem, visit in vids:
            poster = f' poster="/{page}/render/{under(cdir, "docs/clips")}/{stem}-poster.jpg"' if os.path.isfile(os.path.join(run, f"{stem}-poster.jpg")) else ""
            md += [f'::adom-video{{video="/{page}/render/{under(cdir, "docs/clips")}/{stem}-10x.mp4"{poster} title="{title}, visit {visit} (10x)" muted="true"}}', ""]
    t = tokens_by_step(data).get(step)
    if t:
        md += ["## AI usage (Google Antigravity)", "", f"{t['calls']} model calls: {t['freshIn']:,} fresh input, {t['cachedIn']:,} cached input and {t['out']:,} output tokens.", ""]
    open(os.path.join(d, "README.md"), "w").write("\n".join(md))
    return os.path.relpath(d, stage), title, (flow.get("short") or {}).get(step, does[:140])


def main():
    run, page = sys.argv[1], sys.argv[2]
    data, events = load(run)
    seq, seen = steps_done(data)
    want = sys.argv[3:] or seen
    folder = page_dir(data, page)
    stage = os.path.join(run, "project-page", "steps-stage"); shutil.rmtree(stage, ignore_errors=True); os.makedirs(stage)
    files = []
    for st in want:
        if st not in seen: continue
        rel, _, _ = sub_readme(run, data, events, st, seq.index(st) + 1 if st in seq else 0, stage, folder)
        for f in os.listdir(os.path.join(stage, rel)): files.append(os.path.join(rel, f))
    # the main README's section: every step visited so far, one intro each
    readme_rel = under(folder, "README.md")
    shown = subprocess.run(["adom-wiki", "repo", "show", page, readme_rel], capture_output=True, text=True, stdin=subprocess.DEVNULL)
    cur = shown.stdout if shown.returncode == 0 else ""   # a folder with no README yet starts one
    flow = flow_of(data)
    lines = [BEGIN, "## How this board was built", "", "Every step of the build, in order, each with its own page of evidence (checks, screenshots, the AI's drawings and usage), written by [adom-aiflow](https://wiki.adom.inc/adom/adom-aiflow) as the run went.", ""]
    for st in seen:
        n = seq.index(st) + 1 if st in seq else 0
        title = (flow.get("titles") or {}).get(st, st); short = (flow.get("short") or {}).get(st, "")
        lines += [f"**{n}. {title}**: {short}. [The evidence]({'steps/'}{n:02d}-{st}/README.md)", ""]
    lines.append(END)
    block = "\n".join(lines)
    if BEGIN in cur and END in cur:
        new = cur[:cur.index(BEGIN)] + block + cur[cur.index(END) + len(END):]
    else:
        new = cur.rstrip() + "\n\n" + block + "\n"
    os.makedirs(os.path.dirname(os.path.join(stage, readme_rel)), exist_ok=True)
    open(os.path.join(stage, readme_rel), "w").write(new)
    files.append(readme_rel)
    if os.environ.get("AIFLOW_PROJECT_STEPS_DRY"):
        print(json.dumps({"ok": True, "dry": True, "stage": stage, "steps": want, "files": len(files), "paths": files})); return
    p = subprocess.run(["adom-wiki", "repo", "push", page, "--files", *files, "-m", f"adom-aiflow: step evidence ({', '.join(want)})"], capture_output=True, text=True, cwd=stage, stdin=subprocess.DEVNULL)
    m = re.search(r'"commit":\s*"([0-9a-f]{7,})"', p.stdout)
    print(json.dumps({"ok": bool(m), "commit": m.group(1) if m else None, "steps": want, "files": len(files), "error": None if m else clip(p.stderr or p.stdout, 400)}))


if __name__ == "__main__":
    main()