AI Flow: Reverse Engineering

//! `video build`: the two minute video at the end of a run, made from the run folder alone.
//!
//! The pictures are drawn and the frames made by the Python tool in `tools/video/` (PIL, OpenCV and numpy feeding ffmpeg;
//! no browser). Its source is compiled into this binary and written to `reverse-flow/video/work/tool/` at each build, so
//! an installed binary needs no other file. This module does what only the program can: it computes both scores as the
//! files stand, the step states, the hidden connections and the parts' outlines, writes them to `facts.json` for the
//! tool, asks KiCad for its own picture of the copy when KiCad's command line answers, runs the tool and records the call
//! in the ledger. Every figure the video shows is read at build time: from the run's files by the tool, or from here.
//!
//! A failed build never stops a run: `auto` records the reason and goes on to COMPLETE.
use crate::{confidence, flow, items, kicad_out, report, schcheck, unrouted};
use serde_json::{json, Value};
use std::path::{Path, PathBuf};

/// The tool's files, compiled in.
const TOOL: [(&str, &str); 6] = [
    ("build.py", include_str!("../tools/video/build.py")),
    ("engine.py", include_str!("../tools/video/engine.py")),
    ("rundata.py", include_str!("../tools/video/rundata.py")),
    ("scenes.py", include_str!("../tools/video/scenes.py")),
    ("style.py", include_str!("../tools/video/style.py")),
    ("voice.py", include_str!("../tools/video/voice.py")),
];
pub const DIR: &str = "video";
const STITCH_DIR: &str = "photos/stitch";

fn project_name(flow_dir: &Path) -> String { flow_dir.parent().and_then(|p| p.file_name()).map(|f| f.to_string_lossy().to_string()).unwrap_or_else(|| "run".into()) }

/// The video's file stem: `<project>-run`.
pub fn stem(flow_dir: &Path) -> String { format!("{}-run", project_name(flow_dir)) }

fn mtime(p: &Path) -> Option<std::time::SystemTime> { std::fs::metadata(p).and_then(|m| m.modified()).ok() }

/// What the video was built from, as one key: when it changes, a recorded failure no longer counts and the video is built again.
pub fn input_key(flow_dir: &Path) -> String { crate::sha256(&flow_dir.join("checks/report.ok")) }

/// The record of the last build, or Null.
pub fn record(flow_dir: &Path) -> Value { items::read_json(&flow_dir.join(DIR).join("video.json")).unwrap_or(Value::Null) }

/// Whether a finished video stands that is no older than the report it follows.
pub fn fresh(flow_dir: &Path) -> bool {
    let rec = flow_dir.join(DIR).join("video.json");
    let v = record(flow_dir);
    if v["ok"] != json!(true) { return false; }
    let file = v["file"].as_str().map(|f| flow_dir.parent().unwrap_or(Path::new(".")).join(f));
    if !file.is_some_and(|f| f.is_file()) { return false; }
    match (mtime(&rec), mtime(&flow_dir.join("checks/report.ok"))) { (Some(a), Some(b)) => a >= b, (Some(_), None) => true, _ => false }
}

/// One line for `steps` and `next`: where the video stands.
pub fn state_word(flow_dir: &Path) -> &'static str {
    if fresh(flow_dir) { "done" } else if record(flow_dir)["ok"] == json!(false) { "failed" } else if record(flow_dir).is_object() { "stale" } else { "todo" }
}

/// What only the program can say, for the tool: both scores as the files stand with what limits each, the review's verdict,
/// every step's state, the hidden connections, the parts' outlines, the KiCad board and whether the layer scoring is current.
pub fn facts(flow_dir: &Path, state: &Value, fl: &flow::Flow) -> Value {
    let (s, verdict) = report::current(flow_dir);
    let part = |p: &Option<crate::twoscores::Part>| p.as_ref().map(|p| json!({"score": p.score, "limit": p.limit, "raise": p.raise})).unwrap_or(Value::Null);
    let cats: Vec<Value> = s.cats.iter().map(|c| json!({
        "name": c.category, "photo": part(&c.photo), "bench": part(&c.bench),
        "audit": {"status": c.audit.status, "strata": c.audit.strata.iter().map(|x| json!({"stratum": x["stratum"], "audited": x["audited"], "ok": x["ok"], "population": x["population"]})).collect::<Vec<_>>()},
    })).collect();
    let steps: Vec<Value> = fl.steps.iter().map(|st| {
        let done = st.done.iter().all(|x| flow::exists(flow_dir, x));
        let stale = st.done.iter().any(|x| !flow::stale(flow_dir, x).is_empty());
        json!({"name": st.name, "state": if done { "done" } else if stale { "stale" } else { "todo" }})
    }).collect();
    let nets = items::read_json(&flow_dir.join("nets.json")).ok().and_then(|d| d["nets"].as_array().cloned()).unwrap_or_default();
    let parts = items::read_json(&flow_dir.join("parts.json")).ok().and_then(|d| d["parts"].as_array().cloned()).unwrap_or_default();
    let bodies: Vec<Value> = kicad_out::bodies_of(&kicad_out::cache_dir(flow_dir), &parts).into_iter()
        .map(|b| json!({"ref": b.part, "side": if b.layer == "B.Cu" { "bottom" } else { "top" }, "corners": b.corners().iter().map(|q| json!([q.0, q.1])).collect::<Vec<_>>()})).collect();
    let fid = items::read_json(&flow_dir.join("fidelity.json")).unwrap_or(Value::Null);
    let fid_current = if fid.is_object() { json!(fid["inputs"].as_object().into_iter().flatten().all(|(k, h)| h.as_str().is_some_and(|w| flow::matches_hash(&flow_dir.join(k), w)))) } else { Value::Null };
    json!({
        "project": project_name(flow_dir), "version": env!("CARGO_PKG_VERSION"), "builtAt": crate::ledger::now(),
        "complete": flow::current(fl, flow_dir).is_none(), "steps": steps,
        "scores": {"target": s.target, "photo": s.photo, "bench": s.bench, "verdict": verdict, "line": confidence::overall_line(&s, &verdict), "categories": cats},
        "hidden": unrouted::count(&nets), "bodies": bodies,
        "kicad": {"pcb": state["kicad"]["pcb"], "render": Value::Null, "why": Value::Null},
        "fidelityCurrent": fid_current,
        // what the video must say in plain words: a face whose copper was not read, and a board the photos cut off
        "copperNotRead": crate::notread::faces(flow_dir).iter().map(|f| json!({"side": f.side, "layer": f.layer, "photographed": f.photographed, "short": crate::notread::short(f)})).collect::<Vec<_>>(),
        "photosCut": crate::window::facts(flow_dir),
    })
}

/// KiCad's own picture of the copy's top face, saved as `video/stills/kicad-top.png`: (path relative to the flow folder, or
/// why there is none). Rendered again only when the board is newer than the picture.
pub fn kicad_still(flow_dir: &Path, state: &Value) -> (Option<String>, Option<String>) {
    let Some(pcb) = state["kicad"]["pcb"].as_str().map(PathBuf::from).filter(|p| p.is_file()) else { return (None, Some("no KiCad board is registered: `build` has not run".into())) };
    let rel = format!("{DIR}/stills/kicad-top.png");
    let out = flow_dir.join(&rel);
    if let (Some(a), Some(b)) = (mtime(&out), mtime(&pcb)) { if a >= b { return (Some(rel), None); } }
    let Some(cli) = schcheck::kicad_cli() else { return (None, Some("KiCad's command line did not answer (set ADOM_AIFLOW_KICAD_CLI), so KiCad's own picture of the copy was not rendered".into())) };
    if let Some(d) = out.parent() { let _ = std::fs::create_dir_all(d); }
    let _ = std::fs::remove_file(&out);
    let o = std::process::Command::new(&cli).args(["pcb", "render", "--side", "top", "--width", "3840", "--height", "2160", "--quality", "basic", "--background", "transparent", "--output"]).arg(&out).arg(&pcb).output();
    if out.is_file() { return (Some(rel), None); }
    let said = o.map(|o| String::from_utf8_lossy(&o.stderr).trim().chars().take(200).collect::<String>()).unwrap_or_else(|e| e.to_string());
    (None, Some(format!("KiCad's `pcb render` wrote no picture: {said}")))
}

/// At `init`: when the photos come from a stitcher's folder (a `stitch.json` beside them), keep its record and its small
/// evidence pictures with the run's photos, so the video can show how the photos were joined without the source folder.
pub fn keep_stitch(flow_dir: &Path, sources: &[PathBuf]) -> Option<String> {
    let dir = sources.iter().filter_map(|s| s.parent()).find(|d| d.join("stitch.json").is_file())?.to_path_buf();
    let doc = items::read_json(&dir.join("stitch.json")).ok()?;
    let dst = flow_dir.join(STITCH_DIR);
    std::fs::create_dir_all(&dst).ok()?;
    std::fs::copy(dir.join("stitch.json"), dst.join("stitch.json")).ok()?;
    let mut kept = 1;
    for (_, face) in doc["faces"].as_object().into_iter().flatten() {
        let ev = &face["evidence"];
        for name in [ev["sources"].as_str(), ev["sourcesIndex"]["file"].as_str(), ev["overlap"]["file"].as_str()].into_iter().flatten() {
            // plain file names only: nothing outside the stitcher's folder is copied
            if name.contains('/') || name.contains("..") { continue; }
            if std::fs::copy(dir.join(name), dst.join(name)).is_ok() { kept += 1; }
        }
    }
    Some(format!("the stitcher's record is kept with the photos: reverse-flow/{STITCH_DIR}/ ({kept} files), for the video's photo scene"))
}

/// Where the tool runs from: `ADOM_AIFLOW_VIDEO_TOOL` when set (for working on the tool), else the compiled-in copy,
/// written out for this build.
fn tool_dir(flow_dir: &Path) -> Result<PathBuf, String> {
    if let Some(d) = std::env::var("ADOM_AIFLOW_VIDEO_TOOL").ok().filter(|s| !s.is_empty()).map(PathBuf::from) {
        if d.join("build.py").is_file() { return Ok(d); }
        return Err(format!("ADOM_AIFLOW_VIDEO_TOOL names {}, which has no build.py", d.display()));
    }
    let d = flow_dir.join(DIR).join("work").join("tool");
    std::fs::create_dir_all(&d).map_err(|e| format!("{}: {e}", d.display()))?;
    for (name, text) in TOOL { std::fs::write(d.join(name), text).map_err(|e| format!("{}: {e}", d.join(name).display()))?; }
    Ok(d)
}

/// A short hash of the compiled-in tool, for the ledger: which renderer made this video.
pub fn tool_hash() -> String { confidence::sha256_hex(TOOL.iter().map(|(n, t)| format!("{n}\n{t}")).collect::<Vec<_>>().join("\n").as_bytes()).chars().take(16).collect() }

pub struct Built { pub ok: bool, pub line: String, pub hints: Vec<String>, pub call: Vec<String>, pub seconds: f64 }

fn failed(flow_dir: &Path, why: &str, call: Vec<String>, seconds: f64) -> Built {
    // the tool writes video.json itself when it gets that far; when it could not start, the failure is recorded here
    let rec = flow_dir.join(DIR).join("video.json");
    let stands = items::read_json(&rec).ok().is_some_and(|v| v["ok"] == json!(false) && mtime(&rec).zip(mtime(&flow_dir.join(DIR).join("work/facts.json"))).is_some_and(|(a, b)| a >= b));
    if !stands {
        let _ = std::fs::create_dir_all(flow_dir.join(DIR));
        let _ = std::fs::write(&rec, serde_json::to_string_pretty(&json!({"ok": false, "error": why, "builtAt": crate::ledger::now(), "project": project_name(flow_dir)})).unwrap_or_default() + "\n");
    }
    Built { ok: false, line: format!("the video was not built: {why}"), hints: vec![format!("the tool's output is in reverse-flow/{DIR}/build-log.txt; the run itself is not affected, and `video build` can be run again"), "it needs python3 with numpy, Pillow and OpenCV, and ffmpeg; the voice needs adom-tts and is left out when it does not answer".into()], call, seconds }
}

/// Build the video. Never panics and never exits: the caller decides what a failure means.
pub fn build(flow_dir: &Path, project_dir: &Path, state: &Value, fl: &flow::Flow, no_voice: bool, workers: usize) -> Built {
    let t0 = std::time::Instant::now();
    let vdir = flow_dir.join(DIR);
    if let Err(e) = std::fs::create_dir_all(vdir.join("work")) { return failed(flow_dir, &format!("{}: {e}", vdir.display()), vec![], 0.0); }
    let mut f = facts(flow_dir, state, fl);
    let (still, why) = kicad_still(flow_dir, state);
    f["kicad"]["render"] = still.map(|s| json!({"top": s})).unwrap_or(Value::Null);
    f["kicad"]["why"] = json!(why);
    let facts_path = vdir.join("work/facts.json");
    if let Err(e) = std::fs::write(&facts_path, serde_json::to_string_pretty(&f).unwrap_or_default() + "\n") { return failed(flow_dir, &format!("{}: {e}", facts_path.display()), vec![], 0.0); }
    let tool = match tool_dir(flow_dir) { Ok(d) => d, Err(e) => return failed(flow_dir, &e, vec![], 0.0) };
    // the interpreter: this command's own choice, else the one the stitcher is told to use, else python3
    let py = ["ADOM_AIFLOW_PYTHON", "ADOM_REVERSE_PYTHON"].iter().find_map(|k| std::env::var(k).ok().filter(|s| !s.is_empty())).unwrap_or_else(|| "python3".into());
    let mut call: Vec<String> = vec![py.clone(), tool.join("build.py").display().to_string(), "--run".into(), project_dir.display().to_string(), "--facts".into(), facts_path.display().to_string(),
        "--out".into(), vdir.display().to_string(), "--name".into(), stem(flow_dir), "--workers".into(), workers.clamp(1, 4).to_string()];
    if no_voice { call.push("--no-voice".into()); }
    let out = std::process::Command::new(&call[0]).args(&call[1..]).output();
    let secs = t0.elapsed().as_secs_f64();
    let (status_ok, text) = match out {
        Ok(o) => (o.status.success(), String::from_utf8_lossy(&o.stdout).to_string() + &String::from_utf8_lossy(&o.stderr)),
        Err(e) => return failed(flow_dir, &format!("{py} could not be started: {e}"), call, secs),
    };
    let _ = std::fs::write(vdir.join("build-log.txt"), format!("$ {}\n{text}", call.join(" ")));
    let last = text.lines().rev().find(|l| l.starts_with("VIDEO OK:") || l.starts_with("VIDEO FAILED:")).unwrap_or("").to_string();
    if status_ok && last.starts_with("VIDEO OK:") && record(flow_dir)["ok"] == json!(true) {
        let v = record(flow_dir);
        let mut hints = vec![
            format!("{} scenes, {} s; captions beside it (.vtt, .srt, not burned in), poster {}", v["scenes"].as_array().map(|a| a.len()).unwrap_or(0), v["seconds"], v["poster"].as_str().unwrap_or("")),
            format!("reverse-flow/{DIR}/video.json lists every scene with the files it showed and where each figure was read"),
        ];
        if v["voice"]["used"] != json!(true) { hints.push(format!("no voice, captions only: {}", v["voice"]["why"].as_str().unwrap_or("the voice service did not answer"))); }
        for s in v["skipped"].as_array().into_iter().flatten() { hints.push(format!("scene left out: {}: {}", s["scene"].as_str().unwrap_or(""), s["why"].as_str().unwrap_or(""))); }
        if v["withinTarget"] == json!(false) { hints.push(format!("the video is {} s, outside the 110 to 130 s it aims for (video.json notes say why)", v["seconds"])); }
        return Built { ok: true, line: last.trim_start_matches("VIDEO OK: ").to_string(), hints, call, seconds: secs };
    }
    let why = if last.starts_with("VIDEO FAILED:") { last.trim_start_matches("VIDEO FAILED: ").to_string() } else {
        text.lines().rev().find(|l| !l.trim().is_empty()).map(|l| l.trim().chars().take(240).collect::<String>()).unwrap_or_else(|| "the video tool printed nothing".into())
    };
    failed(flow_dir, &why, call, secs)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn dir(name: &str) -> PathBuf {
        let d = std::env::temp_dir().join(format!("video-test-{name}-{}", std::process::id())).join("reverse-flow");
        let _ = std::fs::remove_dir_all(d.parent().unwrap());
        std::fs::create_dir_all(d.join("checks")).unwrap();
        d
    }

    #[test]
    fn the_tool_is_compiled_in_whole_and_written_out_for_a_build() {
        let d = dir("tool");
        let t = tool_dir(&d).unwrap();
        for (name, text) in TOOL { assert_eq!(std::fs::read_to_string(t.join(name)).unwrap(), text); }
        // every module the entry point imports from its own folder is one of the files shipped
        let names: Vec<&str> = TOOL.iter().map(|(n, _)| n.trim_end_matches(".py")).collect();
        for (name, text) in TOOL {
            for line in text.lines().filter(|l| l.starts_with("import ") || l.starts_with("from ")) {
                let m = line.split_whitespace().nth(1).unwrap_or("").split('.').next().unwrap_or("");
                if ["style", "engine", "voice", "rundata", "scenes", "build"].contains(&m) { assert!(names.contains(&m), "{name} imports {m}, which is not shipped"); }
            }
        }
        // the narration is plain: no long dash anywhere in the tool's text
        for (name, text) in TOOL { assert!(!text.contains('\u{2014}'), "{name} holds a long dash"); }
        assert_eq!(tool_hash().len(), 16);
        std::fs::remove_dir_all(d.parent().unwrap()).ok();
    }

    #[test]
    fn a_video_counts_only_while_it_is_whole_and_no_older_than_the_report() {
        let d = dir("fresh");
        assert!(!fresh(&d));
        assert_eq!(state_word(&d), "todo");
        std::fs::create_dir_all(d.join(DIR)).unwrap();
        std::fs::write(d.join("checks/report.ok"), "{}").unwrap();
        std::thread::sleep(std::time::Duration::from_millis(30));
        std::fs::write(d.join(DIR).join("x-run.mp4"), "x").unwrap();
        std::fs::write(d.join(DIR).join("video.json"), json!({"ok": true, "file": "reverse-flow/video/x-run.mp4"}).to_string()).unwrap();
        assert!(fresh(&d));
        assert_eq!(state_word(&d), "done");
        // the report passes again later: the video is stale
        std::thread::sleep(std::time::Duration::from_millis(30));
        std::fs::write(d.join("checks/report.ok"), "{\"again\": 1}").unwrap();
        assert!(!fresh(&d));
        assert_eq!(state_word(&d), "stale");
        // a record of a failure is not a video, and neither is a record whose file is gone
        std::fs::write(d.join(DIR).join("video.json"), json!({"ok": false, "error": "ffmpeg is not installed on this machine"}).to_string()).unwrap();
        assert!(!fresh(&d));
        assert_eq!(state_word(&d), "failed");
        std::fs::write(d.join(DIR).join("video.json"), json!({"ok": true, "file": "reverse-flow/video/gone.mp4"}).to_string()).unwrap();
        assert!(!fresh(&d));
        std::fs::remove_dir_all(d.parent().unwrap()).ok();
    }

    #[test]
    fn a_build_that_cannot_start_is_recorded_and_says_why() {
        let d = dir("nopython");
        std::fs::write(d.join("state.json"), "{}").unwrap();
        std::env::set_var("ADOM_AIFLOW_PYTHON", "/nonexistent/python3-for-this-test");
        let b = build(&d, d.parent().unwrap(), &json!({}), &flow::load(), true, 1);
        std::env::remove_var("ADOM_AIFLOW_PYTHON");
        assert!(!b.ok);
        assert!(b.line.starts_with("the video was not built: ") && b.line.contains("could not be started"), "{}", b.line);
        let v = record(&d);
        assert_eq!(v["ok"], json!(false));
        assert!(v["error"].as_str().unwrap().contains("could not be started"));
        // the facts were still written from the files as they stand, with both scores and every step
        let f = items::read_json(&d.join(DIR).join("work/facts.json")).unwrap();
        assert!(f["scores"]["photo"].is_number() && f["scores"]["bench"].is_number() && f["scores"]["line"].is_string());
        assert_eq!(f["steps"].as_array().unwrap().len(), flow::load().steps.len());
        assert_eq!(f["complete"], json!(false));
        std::fs::remove_dir_all(d.parent().unwrap()).ok();
    }

    #[test]
    fn the_stitchers_record_is_kept_with_the_photos() {
        let d = dir("stitch");
        let src = d.parent().unwrap().join("mosaic");
        std::fs::create_dir_all(&src).unwrap();
        std::fs::write(src.join("top.jpg"), "x").unwrap();
        assert!(keep_stitch(&d, &[src.join("top.jpg")]).is_none());
        std::fs::write(src.join("stitch.json"), json!({"faces": {"top": {"file": "top.jpg", "evidence": {"sources": "top-sources.jpg", "sourcesIndex": {"file": "top-sources.png"}, "overlap": {"file": "../outside.jpg"}}}}}).to_string()).unwrap();
        std::fs::write(src.join("top-sources.jpg"), "a").unwrap();
        std::fs::write(src.join("top-sources.png"), "b").unwrap();
        let h = keep_stitch(&d, &[src.join("top.jpg")]).unwrap();
        assert!(h.contains("3 files"), "{h}");
        assert!(d.join("photos/stitch/stitch.json").is_file() && d.join("photos/stitch/top-sources.png").is_file());
        std::fs::remove_dir_all(d.parent().unwrap()).ok();
    }
}