AI Flow: Reverse Engineering

//! The optional bench passback. After a run completes, `bench plan` writes an ordered instruction set for a person with the
//! real board and a multimeter, and `bench ingest` reads their answers back. The flow never waits for either.
//!
//! The plan holds, in this order: a continuity probe for every hidden connection (`unrouted`), most settled first, each
//! between two named pads with a picture showing where each pad is; the unmarked parts whose value is unknown; the layer
//! count, when it is not measured; and the other measurements tests.json calls for. Ingest upgrades what an answer settles
//! to `MEASURED`, with the answer recorded beside it, and changes nothing else: the scores move because `confidence`
//! computes them again from the files. docs/bench.md gives the results file format.
use crate::kicad_out::{self, Pad};
use crate::{board, draw, items, ledger, layerstate, twoscores, unrouted};
use image::{imageops, RgbImage};
use serde_json::{json, Value};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::Path;

/// What ingest has recorded, by hash of the file each answer came in.
pub const MEASURED: &str = "bench/measured.json";
const DIR: &str = "bench";
/// A crop is this wide on the board, mm: a few parts round the pad, enough to find it by eye.
const CROP_MM: f64 = 8.0;
const PANEL: u32 = 480;
/// The ring sits this far outside half the pad's narrow side, mm, so it marks one pad and not its neighbours at fine pitch.
const RING_CLEAR_MM: f64 = 0.12;

fn st<'a>(v: &'a Value, k: &str) -> &'a str { v[k].as_str().unwrap_or("") }
fn strs(v: &Value) -> Vec<String> { v.as_array().into_iter().flatten().filter_map(|x| x.as_str().map(str::to_string)).collect() }
fn wr(p: &Path, v: &Value) -> Result<(), String> { if let Some(d) = p.parent() { std::fs::create_dir_all(d).map_err(|e| e.to_string())?; } std::fs::write(p, serde_json::to_string_pretty(v).map_err(|e| e.to_string())? + "\n").map_err(|e| format!("{}: {e}", p.display())) }
fn known(tag: &str) -> bool { matches!(tag, "SEEN" | "MEASURED" | "DATASHEET") }

/// One end of a probe: a pin, where its pad is, and how easy it is to touch.
#[derive(Clone)]
struct End { pin: String, x: f64, y: f64, side: &'static str, reach: f64, note: &'static str, ring: f64 }

/// How easy a pad is to put a probe on. Higher is easier. An order of preference, not a measurement: a plated hole or a test
/// point takes a probe best, a connector or a large pad well, and a pad under its own part (a ball grid, a no-lead package)
/// cannot be touched at all.
fn reach(pad: &Pad, part: &Value) -> (f64, &'static str) {
    let pk = format!("{} {}", st(part, "package"), st(part, "footprint")).to_uppercase();
    let r = st(part, "ref").to_uppercase();
    if ["BGA", "LGA", "WLCSP"].iter().any(|k| pk.contains(k)) { return (-5.0, "under the part: not reachable with a probe, use another pin of the net or skip"); }
    let mut s = (pad.w * pad.h).min(4.0) / 2.0;
    if pad.drill > 0.0 { s += 3.0; }
    if r.starts_with("TP") { s += 3.0; }
    if r.starts_with('J') || r.starts_with('H') || r.starts_with('P') { s += 1.5; }
    if ["QFN", "DFN", "SON"].iter().any(|k| pk.contains(k)) { return (s - 3.0, "a no-lead package: touch the pad's outer edge with a fine probe"); }
    (s, "")
}

fn ends_of(piece: &[String], pads: &HashMap<String, &Pad>, parts: &HashMap<&str, &Value>) -> Vec<End> {
    piece.iter().filter_map(|pin| {
        let pad = pads.get(pin)?;
        let part = parts.get(pin.rsplit_once('.')?.0)?;
        let (reach, note) = reach(pad, part);
        Some(End { pin: pin.clone(), x: pad.x, y: pad.y, side: if pad.on("F.Cu") { "top" } else { "bottom" }, reach, note, ring: (pad.w.min(pad.h) / 2.0 + RING_CLEAR_MM).clamp(0.25, 1.5) })
    }).collect()
}

/// The two pads to touch for one hidden connection: the easiest pad of the far piece, and the easiest pad of the net's
/// main piece, nearest to it among equals.
fn pick(from: &[End], to: &[End]) -> Option<(End, End)> {
    let b = to.iter().max_by(|a, b| a.reach.partial_cmp(&b.reach).unwrap().then(b.pin.cmp(&a.pin)))?.clone();
    let best = from.iter().map(|e| e.reach).fold(f64::MIN, f64::max);
    let d = |e: &End| (e.x - b.x).hypot(e.y - b.y);
    let a = from.iter().filter(|e| e.reach >= best - 0.5).min_by(|p, q| d(p).partial_cmp(&d(q)).unwrap().then(p.pin.cmp(&q.pin)))?.clone();
    Some((a, b))
}

/// The face as the person holding the board sees it. The bottom image is stored turned over to line up with the top, so it
/// is turned back: mirrored left to right when the board was flipped that way, top to bottom otherwise.
fn as_seen(img: &RgbImage, side: &str, flip: &str) -> RgbImage { if side != "bottom" { img.clone() } else if flip == "y" { imageops::flip_vertical(img) } else { imageops::flip_horizontal(img) } }

/// One end's two panels, side by side: the pad close up with a ring on it, and the whole face with a cross where it is.
fn end_panels(face: &RgbImage, e: &End, flip: &str, mark: [u8; 3]) -> RgbImage {
    let (cx, cy) = board::to_px(e.x, e.y);
    let half = (CROP_MM / 2.0 * board::RES) as i64;
    let side_px = (2 * half) as u32;
    let mut close = RgbImage::from_pixel(side_px, side_px, image::Rgb([40, 40, 40]));
    for v in 0..side_px as i64 { for u in 0..side_px as i64 {
        let (x, y) = (cx as i64 - half + u, cy as i64 - half + v);
        if x >= 0 && y >= 0 && (x as u32) < face.width() && (y as u32) < face.height() { close.put_pixel(u as u32, v as u32, *face.get_pixel(x as u32, y as u32)); }
    } }
    // a ring just outside the pad, and four ticks pointing at it from further out, so a small pad is found and still seen
    let (c, r) = (half as f64, e.ring * board::RES);
    draw::circle(&mut close, (c, c), r, mark, 2);
    draw::circle(&mut close, (c, c), r + 2.0, [0, 0, 0], 1);
    for (dx, dy) in [(1.0, 0.0), (-1.0, 0.0), (0.0, 1.0), (0.0, -1.0)] { draw::line(&mut close, (c + dx * (r + 10.0), c + dy * (r + 10.0)), (c + dx * (r + 40.0), c + dy * (r + 40.0)), mark, 3); }
    let close = imageops::resize(&as_seen(&close, e.side, flip), PANEL, PANEL, imageops::FilterType::Triangle);
    let k = (PANEL as f64 / face.width() as f64).min(PANEL as f64 / face.height() as f64);
    let small = imageops::resize(face, ((face.width() as f64 * k) as u32).max(1), ((face.height() as f64 * k) as u32).max(1), imageops::FilterType::Triangle);
    let mut over = RgbImage::from_pixel(PANEL, PANEL, image::Rgb([40, 40, 40]));
    imageops::overlay(&mut over, &small, 0, 0);
    let (ox, oy) = (cx * k, cy * k);
    for (a, b) in [((ox - 14.0, oy), (ox + 14.0, oy)), ((ox, oy - 14.0), (ox, oy + 14.0))] { draw::line(&mut over, a, b, mark, 3); }
    draw::circle(&mut over, (ox, oy), 9.0, mark, 2);
    // only the board's own area is turned back, so the cross stays on its pad
    let mut placed = RgbImage::from_pixel(PANEL, PANEL, image::Rgb([40, 40, 40]));
    let cut = imageops::crop_imm(&over, 0, 0, small.width().min(PANEL), small.height().min(PANEL)).to_image();
    imageops::overlay(&mut placed, &as_seen(&cut, e.side, flip), 0, 0);
    let mut out = RgbImage::from_pixel(2 * PANEL + 6, PANEL, image::Rgb([255, 255, 255]));
    imageops::overlay(&mut out, &close, 0, 0);
    imageops::overlay(&mut out, &placed, (PANEL + 6) as i64, 0);
    out
}

/// A picture for one probe or one part: end A's panels above end B's.
fn picture(faces: &HashMap<&'static str, RgbImage>, ends: &[&End], flip: &str) -> Option<RgbImage> {
    let marks = [[255, 0, 200], [0, 220, 255]];
    let rows: Vec<RgbImage> = ends.iter().enumerate().filter_map(|(i, e)| faces.get(e.side).map(|f| end_panels(f, e, flip, marks[i % 2]))).collect();
    if rows.len() != ends.len() { return None; }
    let mut out = RgbImage::from_pixel(2 * PANEL + 6, rows.len() as u32 * (PANEL + 6) - 6, image::Rgb([255, 255, 255]));
    for (i, r) in rows.iter().enumerate() { imageops::overlay(&mut out, r, 0, (i as u32 * (PANEL + 6)) as i64); }
    Some(out)
}

fn save(img: &RgbImage, path: &Path) -> Result<(), String> {
    if let Some(d) = path.parent() { std::fs::create_dir_all(d).map_err(|e| e.to_string())?; }
    let mut f = std::io::BufWriter::new(std::fs::File::create(path).map_err(|e| e.to_string())?);
    image::codecs::jpeg::JpegEncoder::new_with_quality(&mut f, 85).encode_image(img).map_err(|e| e.to_string())
}

fn where_is(e: &End) -> String { format!("{} on the {} face at ({:.1}, {:.1}) mm from the board's top left corner seen from the top", e.pin, e.side, e.x - board::ORIGIN, e.y - board::ORIGIN) }

/// The probes for the hidden connections, in the order to make them: the one that settles most first.
fn continuity(nets: &[Value], pads: &HashMap<String, &Pad>, parts: &HashMap<&str, &Value>, done: &HashSet<String>) -> (Vec<Value>, Vec<Value>) {
    let per_net: HashMap<String, usize> = nets.iter().map(|n| (st(n, "name").to_string(), unrouted::pieces(n).len().saturating_sub(1))).collect();
    let (mut probes, mut cannot) = (vec![], vec![]);
    for h in unrouted::connections(nets) {
        if done.contains(&h.id) { continue; }
        let (from, to) = (ends_of(&h.from, pads, parts), ends_of(&h.to, pads, parts));
        // what a `joined` answer settles: the connection, every pin of the far piece (its place on this net), and the net itself when this is its last hidden connection
        let last = per_net.get(&h.net).copied().unwrap_or(0) == 1;
        let mut settles: Vec<String> = vec![h.id.clone()];
        settles.extend(h.to.iter().cloned());
        if last { settles.push(format!("net:{}", h.net)); }
        match pick(&from, &to) {
            Some((a, b)) if a.reach > -5.0 && b.reach > -5.0 => probes.push((settles.len(), h.net.clone(), a, b, h, settles)),
            other => {
                let why = if other.is_some() { "the only pads of one piece are under their part, where no probe reaches: an X-ray settles it" } else { "a piece has no pad the copy places (metal no footprint explains): find it on the photo and probe it by hand" };
                cannot.push(json!({"connection": h.id, "net": h.net, "from": h.from, "to": h.to, "tag": h.tag, "why": why}));
            }
        }
    }
    // nets that rest on a pinout rule and nothing seen are probed first: they are the cheapest to settle and the most likely to be wrong
    let by_rule: HashSet<&str> = nets.iter().filter(|n| n["fromPinout"] == json!(true) && st(n, "tag") == "INFERRED").map(|n| st(n, "name")).collect();
    probes.sort_by(|a, b| by_rule.contains(b.1.as_str()).cmp(&by_rule.contains(a.1.as_str())).then(b.0.cmp(&a.0)).then(a.1.cmp(&b.1)).then(a.4.id.cmp(&b.4.id)));
    let rows = probes.into_iter().enumerate().map(|(i, (n, net, a, b, h, settles))| {
        let id = format!("P{:03}", i + 1);
        json!({"probe": id, "kind": "continuity", "connection": h.id, "net": net, "tag": h.tag, "source": h.source, "tests": h.tests, "to": h.to,
            "a": a.pin, "aAt": {"side": a.side, "mm": [a.x, a.y], "note": a.note, "ringMm": a.ring}, "b": b.pin, "bAt": {"side": b.side, "mm": [b.x, b.y], "note": b.note, "ringMm": b.ring},
            "settles": settles, "settlesCount": n, "expect": "joined", "picture": format!("{DIR}/crops/{id}.jpg"),
            "do": format!("Board unpowered. Continuity from {} to {}.", where_is(&a), where_is(&b)),
            "ifOpen": format!("the pin map is wrong here: {} is not on {net} with {}. Record `open`; the flow keeps the connection unproven and flags it", b.pin, a.pin)})
    }).collect();
    (rows, cannot)
}

/// `bench plan`: the instruction set, its pictures, and a results file ready to fill in.
pub fn plan(flow: &Path) -> Result<(Value, Vec<String>), String> {
    let nets: Vec<Value> = items::read_json(&flow.join("nets.json")).map_err(|e| format!("{e}: the plan is made from the nets, after the connect step"))?["nets"].as_array().cloned().unwrap_or_default();
    let parts_doc = items::read_json(&flow.join("parts.json"))?;
    let parts: Vec<Value> = parts_doc["parts"].as_array().cloned().unwrap_or_default().into_iter().filter(|p| p["onBoard"] != json!(false)).collect();
    let pad_list = kicad_out::pads_of(&kicad_out::cache_dir(flow), &parts);
    let mut pads: HashMap<String, &Pad> = HashMap::new();
    for p in &pad_list { pads.entry(format!("{}.{}", p.part, p.num)).or_insert(p); }
    let by_ref: HashMap<&str, &Value> = parts.iter().map(|p| (st(p, "ref"), p)).collect();
    let done: HashSet<String> = unrouted::measured_now(flow, &nets).into_iter().map(|h| h.id).collect();
    let (probes, cannot) = continuity(&nets, &pads, &by_ref, &done);
    // unmarked parts whose value nothing settles yet: an in-circuit reading is only a lower bound for a capacitor and an upper one for a resistor, so the method says to lift one end when it matters
    let values: Vec<Value> = parts.iter().filter(|p| (!p["fieldTags"]["value"].is_null() || !st(p, "value").is_empty()) && twoscores::value_stratum(p) == "unmarked" && !known(p["fieldTags"]["value"].as_str().unwrap_or(st(p, "tag")))).map(|p| {
        let r = st(p, "ref");
        json!({"value": r, "package": p["package"], "function": p["function"], "stated": p["value"], "side": p["side"], "mm": p["pos_mm"], "picture": format!("{DIR}/crops/V-{r}.jpg"),
            "do": format!("Measure {r} ({}, {}) on the {} face: LCR meter across it for a capacitor or inductor, ohms for a resistor. Board unpowered. Parts in parallel read together: lift one end, or record `in circuit` in the method", st(p, "package"), if st(p, "function").is_empty() { "function not stated" } else { st(p, "function") }, st(p, "side"))})
    }).collect();
    let layers = items::read_json(&flow.join("layers.json")).unwrap_or(json!({}));
    let layers_row = (!matches!(st(&layers, "tag"), "MEASURED")).then(|| json!({"layers": "the copper layer count", "stated": layers["proposed"], "tag": layers["tag"], "fromThePhotos": layers["minimum"]["why"],
        "do": "Count the copper layers on a cut or sanded board edge under a microscope (each copper layer is a bright line), or X-ray the board. Photograph what you counted and give the file as `evidence`", "methods": layerstate::MEASURED_KINDS}));
    // everything else tests.json calls for, grouped by method: continuity of nets is the probes above, and a value or the layer count is its own row
    let covered: HashSet<String> = values.iter().map(|v| st(v, "value").to_string()).collect();
    let mut groups: BTreeMap<String, (Value, Vec<String>, Vec<String>)> = BTreeMap::new();
    for t in items::read_json(&flow.join("tests.json")).ok().and_then(|t| t["tests"].as_array().cloned()).unwrap_or_default() {
        let settles = strs(&t["settles"]);
        if settles.iter().all(|s| s.starts_with("net:") || s == "layers" || s.starts_with("plane:") || covered.contains(s)) { continue; }
        let g = groups.entry(st(&t, "method").to_string()).or_insert_with(|| (json!({"method": t["method"], "equipment": t["equipment"], "expected": t["expected"], "ifNot": t["ifNot"]}), vec![], vec![]));
        g.1.push(st(&t, "id").to_string()); g.2.extend(settles);
    }
    let mut other: Vec<Value> = groups.into_values().map(|(mut g, ids, settles)| { g["tests"] = json!(ids); g["settles"] = json!(settles); g["settlesCount"] = json!(settles.len()); g }).collect();
    other.sort_by(|a, b| b["settlesCount"].as_u64().cmp(&a["settlesCount"].as_u64()));

    // pictures: where each pad is, as the person holding the board sees it
    let flip = items::read_json(&flow.join("scale.json")).ok().and_then(|s| s["frame"]["bottomFlip"].as_str().map(str::to_string)).unwrap_or_else(|| "x".into());
    let mut faces: HashMap<&'static str, RgbImage> = HashMap::new();
    for (side, file) in [("top", "board/top.jpg"), ("bottom", "board/bottom.jpg")] { if let Ok(i) = image::open(flow.join(file)) { faces.insert(side, i.to_rgb8()); } }
    let dir = flow.join(DIR);
    let _ = std::fs::remove_dir_all(dir.join("crops"));
    let mut drawn = 0;
    let end_at = |v: &Value, pin: &str| End { pin: pin.to_string(), x: v["mm"][0].as_f64().unwrap_or(0.0), y: v["mm"][1].as_f64().unwrap_or(0.0), side: if st(v, "side") == "bottom" { "bottom" } else { "top" }, reach: 0.0, note: "", ring: v["ringMm"].as_f64().unwrap_or(0.9) };
    for p in &probes {
        let (a, b) = (end_at(&p["aAt"], st(p, "a")), end_at(&p["bAt"], st(p, "b")));
        if let Some(img) = picture(&faces, &[&a, &b], &flip) { save(&img, &flow.join(st(p, "picture")))?; drawn += 1; }
    }
    for v in &values {
        let e = End { pin: st(v, "value").to_string(), x: v["mm"][0].as_f64().unwrap_or(0.0), y: v["mm"][1].as_f64().unwrap_or(0.0), side: if st(v, "side") == "bottom" { "bottom" } else { "top" }, reach: 0.0, note: "", ring: 0.9 };
        if let Some(img) = picture(&faces, &[&e], &flip) { save(&img, &flow.join(st(v, "picture")))?; drawn += 1; }
    }

    let key: Vec<String> = probes.iter().map(|p| format!("{}|{}|{}|{}", st(p, "probe"), st(p, "connection"), st(p, "a"), st(p, "b"))).collect();
    let plan_hash = crate::confidence::sha256_hex(key.join("\n").as_bytes())[..16].to_string();
    let name = flow.parent().and_then(|p| p.file_name()).map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
    let settled: usize = probes.iter().map(|p| p["settlesCount"].as_u64().unwrap_or(0) as usize).sum();
    let doc = json!({"board": name, "planHash": plan_hash, "at": ledger::now(), "bottomShown": format!("as you see it with the board turned over {}", if flip == "y" { "top to bottom" } else { "left to right" }),
        "continuity": probes, "cannotBeProbed": cannot, "values": values, "layers": layers_row, "other": other});
    wr(&dir.join("plan.json"), &doc)?;
    // a results file ready to fill in
    let mut rows: Vec<Value> = probes.iter().map(|p| json!({"probe": p["probe"], "result": "", "ohms": null, "note": format!("{} to {}", st(p, "a"), st(p, "b"))})).collect();
    rows.extend(values.iter().map(|v| json!({"value": v["value"], "measured": "", "method": ""})));
    if layers_row.is_some() { rows.push(json!({"layers": null, "method": "", "evidence": ""})); }
    wr(&dir.join("results-template.json"), &json!({"board": name, "planHash": plan_hash, "by": "", "date": "", "instrument": "", "results": rows}))?;

    let mut md = format!("# Bench plan: {name}\n\nWritten by `adom-aiflow-reverse bench plan` from the run's files. It is optional: the run is complete without it. Each answer you record settles something the photos could not, and `bench ingest` turns it into `MEASURED`.\n\n");
    md += "## Before you start\n\n- The board is unpowered and nothing is plugged into it. Discharge large capacitors.\n- A multimeter with a continuity beeper and fine probes. An LCR meter for the part values. A microscope for the layer count.\n";
    md += &format!("- Positions are in mm from the board's top left corner, seen from the top. Pictures of the bottom face are shown {}.\n", st(&doc, "bottomShown"));
    md += "- Each picture has, for each pad, a close view 8 mm across with a ring round the pad and four ticks pointing at it, and the whole face with a cross on it. Pink is pad A, blue is pad B. On a fine connector the ring marks one pin: count from the end of the row to be sure.\n";
    md += &format!("- Record answers in a copy of `reverse-flow/{DIR}/results-template.json` (format: docs/bench.md), then `adom-aiflow-reverse bench ingest --file <your file>`. Answer what you measured and leave the rest: a partial file is fine.\n- A probe answer is `joined` (the meter beeps, a few ohms at most), `open`, or `not_measured`.\n");
    md += &format!("\n## 1. Continuity probes ({} probes, settling {settled} items)\n\nIn this order: the probe that settles the most inferred or unknown items comes first.\n\n| Probe | Net | Pad A | Pad B | Settles | Now | Picture |\n|---|---|---|---|---|---|---|\n", probes.len());
    for p in &probes {
        let note = |k: &str| { let n = st(&p[k], "note"); if n.is_empty() { String::new() } else { format!(" ({n})") } };
        md += &format!("| {} | {} | {}, {} face, ({:.1}, {:.1}){} | {}, {} face, ({:.1}, {:.1}){} | {} | {} | [{}](crops/{}.jpg) |\n", st(p, "probe"), st(p, "net"),
            st(p, "a"), st(&p["aAt"], "side"), p["aAt"]["mm"][0].as_f64().unwrap_or(0.0) - board::ORIGIN, p["aAt"]["mm"][1].as_f64().unwrap_or(0.0) - board::ORIGIN, note("aAt"),
            st(p, "b"), st(&p["bAt"], "side"), p["bAt"]["mm"][0].as_f64().unwrap_or(0.0) - board::ORIGIN, p["bAt"]["mm"][1].as_f64().unwrap_or(0.0) - board::ORIGIN, note("bAt"),
            p["settlesCount"], st(p, "tag"), st(p, "probe"), st(p, "probe"));
    }
    if !cannot.is_empty() {
        md += &format!("\n{} hidden connection(s) have no pad a probe can reach:\n\n", cannot.len());
        for c in &cannot { md += &format!("- {} ({}): {}\n", st(c, "net"), strs(&c["to"]).join(", "), st(c, "why")); }
    }
    md += &format!("\n## 2. Values of unmarked parts ({} parts)\n\nA photo cannot show the value of an unmarked part. Board unpowered. A part with others in parallel reads together with them: lift one end, or say `in circuit` in the method.\n\n| Part | Package | Face | At (mm) | Stated now | Picture |\n|---|---|---|---|---|---|\n", values.len());
    for v in &values { md += &format!("| {} | {} | {} | ({:.1}, {:.1}) | {} | [{}](crops/V-{}.jpg) |\n", st(v, "value"), st(v, "package"), st(v, "side"), v["mm"][0].as_f64().unwrap_or(0.0) - board::ORIGIN, v["mm"][1].as_f64().unwrap_or(0.0) - board::ORIGIN, if st(v, "stated").is_empty() { "nothing" } else { st(v, "stated") }, st(v, "value"), st(v, "value")); }
    md += "\n## 3. Layer count\n\n";
    md += &match &layers_row { Some(l) => format!("Stated now: {} ({}). From the photos: {}\n\n{}. Methods: {}.\n", if l["stated"].is_null() { "unknown".to_string() } else { format!("{} layers", l["stated"]) }, st(l, "tag"), st(l, "fromThePhotos"), st(l, "do"), layerstate::MEASURED_KINDS.join(", ")), None => "Already measured.\n".to_string() };
    md += "\n## 4. Other measurements the tests call for\n\nRecorded as given (`{\"test\": \"<id>\", \"result\": \"...\"}`); the item they settle is corrected by whoever runs the flow.\n\n| Method | Equipment | Expected | Settles | Tests |\n|---|---|---|---|---|\n";
    for g in &other { let ids = strs(&g["tests"]); md += &format!("| {} | {} | {} | {} items | {}{} |\n", st(g, "method"), st(g, "equipment"), st(g, "expected"), g["settlesCount"], ids.iter().take(8).cloned().collect::<Vec<_>>().join(", "), if ids.len() > 8 { format!(" and {} more (plan.json)", ids.len() - 8) } else { String::new() }); }
    std::fs::write(dir.join("plan.md"), md).map_err(|e| e.to_string())?;
    let hints = vec![
        format!("{} continuity probes in order of what each settles ({settled} items in all), {} unmarked part values, {}; {drawn} pictures in reverse-flow/{DIR}/crops/", probes.len(), values.len(), if layers_row.is_some() { "the layer count" } else { "the layer count is already measured" }),
        format!("{} hidden connection(s) have no pad a probe can reach (plan.json, cannotBeProbed)", cannot.len()),
        format!("fill in a copy of reverse-flow/{DIR}/results-template.json (docs/bench.md), then `bench ingest --file <it>`. The flow does not wait for this"),
    ];
    Ok((doc, hints))
}

/// Stamp the recorded probe answers on the nets: the answers themselves, and `MEASURED` on a net whose every hidden connection
/// was found joined. Run after ingest, and after `connect attribute`, which writes the nets' tags again from the copper.
pub fn reapply(flow: &Path) -> usize {
    let Ok(mut doc) = items::read_json(&flow.join("nets.json")) else { return 0 };
    let nets: Vec<Value> = doc["nets"].as_array().cloned().unwrap_or_default();
    let all = unrouted::measured(flow);
    if all.is_empty() && !nets.iter().any(|n| !n["benchMeasured"].is_null()) { return 0; }
    let joined: HashMap<String, Value> = unrouted::measured_now(flow, &nets).into_iter().filter_map(|h| all.get(&h.id).map(|v| (h.id.clone(), v.clone()))).collect();
    let mut upgraded = 0;
    for n in doc["nets"].as_array_mut().into_iter().flatten() {
        let name = st(n, "name").to_string();
        let pieces = unrouted::pieces(n);
        let ids: Vec<String> = pieces.iter().skip(1).map(|p| format!("hc:{name}:{}", p[0])).collect();
        let have: Vec<&Value> = ids.iter().filter_map(|i| joined.get(i)).collect();
        if have.is_empty() { if let Some(o) = n.as_object_mut() { o.remove("benchMeasured"); } continue; }
        n["benchMeasured"] = json!(have);
        if have.len() == ids.len() {
            n["tag"] = json!("MEASURED");
            n["evidence"] = json!([MEASURED]);
            upgraded += 1;
        }
    }
    let _ = wr(&flow.join("nets.json"), &doc);
    upgraded
}

/// `bench ingest`: read a results file, record what it settles, upgrade those items to `MEASURED`. An answer is refused, with
/// the reason, when its probe is not in the plan or the connection it was planned for is no longer in nets.json.
pub fn ingest(flow: &Path, file: &Path) -> Result<(Value, Vec<String>), String> {
    let text = std::fs::read(file).map_err(|e| format!("{}: {e}", file.display()))?;
    let res: Value = serde_json::from_slice(&text).map_err(|e| format!("{}: not valid JSON: {e}", file.display()))?;
    let by = st(&res, "by").trim().to_string();
    if by.is_empty() { return Err("the results file needs `by`: who measured".into()); }
    let rows = res["results"].as_array().cloned().ok_or("the results file needs `results`: a list of answers")?;
    let plan = items::read_json(&flow.join(DIR).join("plan.json")).map_err(|_| "no bench plan: run `bench plan` first, so each probe has a number".to_string())?;
    if let Some(h) = res["planHash"].as_str() { if h != st(&plan, "planHash") { return Err(format!("the results were written for plan {h}, and the current plan is {}: the probe numbers differ. Use the plan the results were measured with, or measure again", st(&plan, "planHash"))); } }
    let sha = crate::confidence::sha256_hex(&text);
    let kept = flow.join(DIR).join("ingested").join(format!("{}.json", &sha[..12]));
    std::fs::create_dir_all(kept.parent().unwrap()).map_err(|e| e.to_string())?;
    std::fs::write(&kept, &text).map_err(|e| e.to_string())?;
    let stamp = json!({"by": by, "date": res["date"], "instrument": res["instrument"], "file": format!("{DIR}/ingested/{}.json", &sha[..12]), "sha256": sha, "ingestedAt": ledger::now()});
    let mut m = items::read_json(&flow.join(MEASURED)).unwrap_or(json!({}));
    for k in ["connections", "values", "tests"] { if !m[k].is_object() { m[k] = json!({}); } }
    if !m["files"].is_array() { m["files"] = json!([]); }
    if !m["files"].as_array().unwrap().iter().any(|f| f["sha256"] == stamp["sha256"]) { m["files"].as_array_mut().unwrap().push(stamp.clone()); }

    let nets: Vec<Value> = items::read_json(&flow.join("nets.json"))?["nets"].as_array().cloned().unwrap_or_default();
    let live: HashMap<String, unrouted::Hidden> = unrouted::connections(&nets).into_iter().map(|h| (h.id.clone(), h)).collect();
    let probes: HashMap<&str, &Value> = plan["continuity"].as_array().into_iter().flatten().map(|p| (st(p, "probe"), p)).collect();
    let mut parts_doc = items::read_json(&flow.join("parts.json"))?;
    let (mut joined, mut open, mut skipped, mut values, mut layers, mut tests) = (0, 0, 0, 0, 0, 0);
    let (mut refused, mut hints): (Vec<String>, Vec<String>) = (vec![], vec![]);
    for r in &rows {
        if let Some(id) = r["probe"].as_str() {
            let Some(p) = probes.get(id) else { refused.push(format!("{id}: not a probe of the current plan")); continue };
            let hc = st(p, "connection");
            match live.get(hc) { Some(h) if h.to == strs(&p["to"]) => {}, _ => { refused.push(format!("{id}: the connection it was planned for ({hc}) is no longer in nets.json as planned: run `bench plan` again")); continue; } }
            let rec = |result: &str| { let mut v = stamp.clone(); v["result"] = json!(result); v["probe"] = json!(id); v["net"] = p["net"].clone(); v["a"] = p["a"].clone(); v["b"] = p["b"].clone(); v["to"] = p["to"].clone(); v["ohms"] = r["ohms"].clone(); v["note"] = r["note"].clone(); v };
            match st(r, "result") {
                "joined" => { m["connections"][hc] = rec("joined"); joined += 1; }
                "open" => { m["connections"][hc] = rec("open"); open += 1; hints.push(format!("{id}: {} and {} are NOT joined, so the pin map is wrong on {}: the connection stays unproven. Correct nets.json from what the board shows", st(p, "a"), st(p, "b"), st(p, "net"))); }
                "" | "not_measured" => skipped += 1,
                other => refused.push(format!("{id}: result must be joined, open or not_measured, not {other:?}")),
            }
        } else if let Some(part) = r["value"].as_str() {
            let measured = st(r, "measured").trim();
            if measured.is_empty() { skipped += 1; continue; }
            let Some(pt) = parts_doc["parts"].as_array_mut().and_then(|a| a.iter_mut().find(|p| st(p, "ref") == part)) else { refused.push(format!("{part}: no such part in parts.json")); continue };
            let was = pt["value"].clone();
            pt["value"] = json!(measured);
            pt["fieldTags"]["value"] = json!("MEASURED");
            pt["valueSource"] = json!(format!("bench: {}, by {by}{}", if st(r, "method").is_empty() { "method not stated" } else { st(r, "method") }, res["date"].as_str().map(|d| format!(", {d}")).unwrap_or_default()));
            let mut v = stamp.clone(); v["measured"] = json!(measured); v["method"] = r["method"].clone(); v["was"] = was;
            m["values"][part] = v; values += 1;
        } else if !r["layers"].is_null() {
            let Some(n) = r["layers"].as_u64().filter(|n| *n > 0) else { skipped += 1; continue };
            let kind = st(r, "method");
            if !layerstate::MEASURED_KINDS.contains(&kind) { refused.push(format!("layers: method must be one of {}, not {kind:?}", layerstate::MEASURED_KINDS.join(", "))); continue; }
            // the record of the reading is the results file itself, unless a picture of what was counted came with it
            let ev = r["evidence"].as_str().filter(|e| !e.trim().is_empty()).map(|e| if Path::new(e).is_absolute() { e.to_string() } else { file.parent().map(|d| d.join(e).display().to_string()).unwrap_or_else(|| e.to_string()) }).unwrap_or_else(|| kept.display().to_string());
            match layerstate::state(flow, &layerstate::Ask { count: Some(n as usize), source: Some(&format!("bench: {kind}, by {by}")), evidence: Some(&ev), kind: Some(kind) }) {
                Ok(_) => { m["layers"] = { let mut v = stamp.clone(); v["layers"] = json!(n); v["method"] = json!(kind); v["evidence"] = json!(ev); v }; layers += 1;
                    hints.push(format!("layers: {n} stated as MEASURED. The layers score rises past its cap only when someone who did not make this run judges that evidence: `audit draw --category layers`, then `audit verdict --category layers --sample <seed> --item a001 --verdict ok --who <auditor> --evidence {ev} --kind {kind}`")); }
                Err(e) => refused.push(format!("layers: {e}")),
            }
        } else if let Some(t) = r["test"].as_str() {
            let mut v = stamp.clone(); v["result"] = r["result"].clone(); v["note"] = r["note"].clone();
            m["tests"][t] = v; tests += 1;
        } else { refused.push(format!("an answer with none of `probe`, `value`, `layers` or `test`: {r}")); }
    }
    wr(&flow.join(MEASURED), &m)?;
    if values > 0 { wr(&flow.join("parts.json"), &parts_doc)?; }
    let upgraded = reapply(flow);
    if tests > 0 { hints.push(format!("{tests} other test result(s) are recorded in {MEASURED} as given: correct the items they settle yourself, and say how each is known")); }
    if skipped > 0 { hints.push(format!("{skipped} answer(s) were left empty or not measured: nothing recorded for them")); }
    if upgraded > 0 { hints.push(format!("{upgraded} net(s) now have every hidden connection measured: tagged MEASURED in nets.json")); }
    for r in &refused { hints.push(format!("refused: {r}")); }
    Ok((json!({"file": stamp["file"], "sha256": stamp["sha256"], "by": by, "joined": joined, "open": open, "values": values, "layers": layers, "tests": tests, "skipped": skipped, "refused": refused.len(), "netsUpgraded": upgraded}), hints))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn end(pin: &str, x: f64, reach: f64) -> End { End { pin: pin.into(), x, y: 0.0, side: "top", reach, note: "", ring: 0.5 } }

    #[test]
    fn a_probe_takes_the_easiest_pad_of_each_piece_and_the_nearer_one_among_equals() {
        let from = vec![end("C1.1", 0.0, 1.0), end("TP1.1", 50.0, 4.0), end("TP2.1", 9.0, 4.0)];
        let to = vec![end("U1.7", 10.0, 0.2), end("J1.3", 12.0, 2.5)];
        let (a, b) = pick(&from, &to).unwrap();
        assert_eq!(b.pin, "J1.3");
        assert_eq!(a.pin, "TP2.1");
        assert!(pick(&from, &[]).is_none());
    }

    #[test]
    fn a_pad_under_its_part_cannot_be_probed_and_a_plated_hole_is_preferred() {
        let pad = |drill: f64| Pad { part: "X".into(), num: "1".into(), x: 0.0, y: 0.0, w: 1.0, h: 1.0, rot: 0.0, round: false, drill, layers: vec!["F.Cu".into()], net: String::new() };
        assert!(reach(&pad(0.0), &json!({"ref": "U1", "package": "BGA"})).0 <= -5.0);
        assert!(reach(&pad(0.8), &json!({"ref": "J1", "package": "PinHeader"})).0 > reach(&pad(0.0), &json!({"ref": "C1", "package": "C0402"})).0);
        assert!(reach(&pad(0.0), &json!({"ref": "TP3", "package": "TestPoint"})).0 > reach(&pad(0.0), &json!({"ref": "R1", "package": "R0402"})).0);
    }

    #[test]
    fn probes_are_ordered_by_what_each_settles() {
        let pads_v = vec![("R1", "1"), ("R2", "1"), ("R3", "1"), ("R4", "1"), ("R5", "1"), ("R6", "1")].into_iter().enumerate().map(|(i, (p, n))| Pad { part: p.into(), num: n.into(), x: 100.0 + i as f64, y: 100.0, w: 0.5, h: 0.5, rot: 0.0, round: false, drill: 0.0, layers: vec!["F.Cu".into()], net: String::new() }).collect::<Vec<_>>();
        let pads: HashMap<String, &Pad> = pads_v.iter().map(|p| (format!("{}.{}", p.part, p.num), p)).collect();
        let parts_v: Vec<Value> = (1..=6).map(|i| json!({"ref": format!("R{i}"), "package": "R0402"})).collect();
        let parts: HashMap<&str, &Value> = parts_v.iter().map(|p| (st(p, "ref"), p)).collect();
        // net A: one hidden connection to a lone pin. Net B: a main piece and a far piece of two pins, its only hidden connection
        let nets = vec![json!({"name": "A", "pins": ["R1.1", "R2.1", "R3.1"], "copperGroups": [["R1.1", "R2.1"]], "tag": "INFERRED"}),
                        json!({"name": "B", "pins": ["R4.1", "R5.1", "R6.1"], "copperGroups": [["R5.1", "R6.1"]], "tag": "INFERRED"})];
        let (probes, cannot) = continuity(&nets, &pads, &parts, &HashSet::new());
        assert!(cannot.is_empty());
        assert_eq!(probes.len(), 2);
        // both settle the connection, one pin and the net: equal counts fall back to the net's name
        assert_eq!(probes[0]["net"], "A");
        assert_eq!(probes[0]["settlesCount"], 3);
        assert_eq!(probes[0]["probe"], "P001");
        // a net with two hidden connections does not have its net settled by one probe, so its probes come later
        let nets2 = vec![json!({"name": "A", "pins": ["R1.1", "R2.1", "R3.1"], "copperGroups": [], "tag": "INFERRED"}), nets[1].clone()];
        let (p2, _) = continuity(&nets2, &pads, &parts, &HashSet::new());
        assert_eq!(p2.len(), 3);
        assert_eq!(p2[0]["net"], "B");
        assert_eq!(p2[0]["settlesCount"], 3);
        assert_eq!(p2[1]["settlesCount"], 2);
        // a connection already measured is not planned again
        let (p3, _) = continuity(&nets, &pads, &parts, &["hc:A:R3.1".to_string()].into_iter().collect());
        assert_eq!(p3.len(), 1);
    }

    #[test]
    fn ingest_records_answers_upgrades_what_they_settle_and_refuses_what_is_stale() {
        let d = std::env::temp_dir().join(format!("bench-test-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&d);
        std::fs::create_dir_all(d.join(DIR)).unwrap();
        let put = |f: &str, v: Value| std::fs::write(d.join(f), v.to_string()).unwrap();
        put("nets.json", json!({"nets": [{"name": "A", "pins": ["R1.1", "R2.1", "R3.1"], "copperGroups": [["R1.1", "R2.1"]], "tag": "INFERRED", "tests": ["T-net-A"]}, {"name": "B", "pins": ["R4.1", "R5.1"], "copperGroups": [], "tag": "INFERRED", "tests": ["T-net-B"]}]}));
        put("parts.json", json!({"parts": [{"ref": "C1", "value": "", "marking": "", "tag": "MEASURED", "fieldTags": {"value": "UNKNOWN"}}]}));
        put("bench/plan.json", json!({"planHash": "abc", "continuity": [
            {"probe": "P001", "connection": "hc:A:R3.1", "net": "A", "a": "R1.1", "b": "R3.1", "to": ["R3.1"]},
            {"probe": "P002", "connection": "hc:B:R5.1", "net": "B", "a": "R4.1", "b": "R5.1", "to": ["R5.1"]},
            {"probe": "P003", "connection": "hc:GONE:R9.1", "net": "GONE", "a": "R8.1", "b": "R9.1", "to": ["R9.1"]}]}));
        let results = d.join("results.json");
        std::fs::write(&results, json!({"by": "a person", "date": "2026-11-11", "instrument": "DMM", "planHash": "abc", "results": [
            {"probe": "P001", "result": "joined", "ohms": 0.2}, {"probe": "P002", "result": "open"}, {"probe": "P003", "result": "joined"}, {"probe": "P999", "result": "joined"},
            {"value": "C1", "measured": "100nF", "method": "LCR, one end lifted"}, {"value": "C2", "measured": "1uF"}, {"probe": "P001x", "result": ""}, {"test": "T-U2", "result": "marking reads AB12"}]}).to_string()).unwrap();
        let (sum, hints) = ingest(&d, &results).unwrap();
        assert_eq!((sum["joined"].as_u64(), sum["open"].as_u64(), sum["values"].as_u64(), sum["tests"].as_u64(), sum["refused"].as_u64()), (Some(1), Some(1), Some(1), Some(1), Some(4)), "{sum} {hints:?}");
        let nets = items::read_json(&d.join("nets.json")).unwrap();
        // every hidden connection of A was found joined: the net is measured, and says by what
        assert_eq!(nets["nets"][0]["tag"], "MEASURED");
        assert_eq!(nets["nets"][0]["benchMeasured"][0]["probe"], "P001");
        assert_eq!(nets["nets"][0]["benchMeasured"][0]["by"], "a person");
        // B's probe read open: nothing is upgraded, and the answer is kept
        assert_eq!(nets["nets"][1]["tag"], "INFERRED");
        let m = items::read_json(&d.join(MEASURED)).unwrap();
        assert_eq!(m["connections"]["hc:B:R5.1"]["result"], "open");
        assert_eq!(unrouted::measured_now(&d, nets["nets"].as_array().unwrap()).len(), 1);
        let parts = items::read_json(&d.join("parts.json")).unwrap();
        assert_eq!(parts["parts"][0]["value"], "100nF");
        assert_eq!(parts["parts"][0]["fieldTags"]["value"], "MEASURED");
        assert!(parts["parts"][0]["valueSource"].as_str().unwrap().contains("LCR"));
        // attribution writes the tags again from the copper; the recorded answer is stamped back
        let mut again = nets.clone(); again["nets"][0]["tag"] = json!("INFERRED"); again["nets"][0].as_object_mut().unwrap().remove("benchMeasured");
        put("nets.json", again);
        assert_eq!(reapply(&d), 1);
        assert_eq!(items::read_json(&d.join("nets.json")).unwrap()["nets"][0]["tag"], "MEASURED");
        // results written for another plan are refused whole
        std::fs::write(&results, json!({"by": "a person", "planHash": "other", "results": []}).to_string()).unwrap();
        assert!(ingest(&d, &results).err().unwrap().contains("plan"));
        std::fs::remove_dir_all(&d).ok();
    }

    #[test]
    fn a_bottom_pad_is_shown_turned_back_as_the_holder_sees_it() {
        let mut img = RgbImage::from_pixel(8, 4, image::Rgb([0, 0, 0]));
        img.put_pixel(0, 0, image::Rgb([255, 0, 0]));
        assert_eq!(as_seen(&img, "top", "x").get_pixel(0, 0)[0], 255);
        assert_eq!(as_seen(&img, "bottom", "x").get_pixel(7, 0)[0], 255);
        assert_eq!(as_seen(&img, "bottom", "y").get_pixel(0, 3)[0], 255);
    }
}