AI Flow: Reverse Engineering

//! The layer count, stated without routing. `layers state` writes layers.json from what is known and nothing else:
//!
//! - a count from outside evidence, given with `--count` and either `--source` (a document) or `--evidence <file> --kind ...`
//!   (a measurement on the board itself: an edge micrograph, a cross-section, an X-ray or a CT scan);
//! - otherwise the count is `UNKNOWN`, with its test, and beside it the minimum the photos prove or suggest.
//!
//! Photos show two faces and never an inner layer, so they can only bound the count from below. Routing difficulty is no
//! evidence of a layer count either (see docs/method.md), which is why the default flow no longer routes to find one.
use crate::checks::{Outcome, Pass};
use crate::items;
use serde_json::{json, Value};
use std::path::Path;

/// A via counts as ending inside the board only when the other face shows no candidate of any kind this close to it, mm.
/// The via detectors place a dome within about 0.1 mm of its pair on the other face (`domes::pits_confirmed` uses 0.1 mm), so
/// three times that leaves no room for a pair that was read a little off. Policy, not fitted to a board.
const OTHER_FACE_CLEAR_MM: f64 = 0.3;
/// Fewer one-face vias than this prove nothing: a handful can be detector misses. Policy, not fitted to a board.
const ONE_FACE_MIN: usize = 10;
/// And the two faces must show at least this many vias in the same place, or the photos have not shown that a through via
/// appears on both. Policy, not fitted to a board.
const BOTH_FACES_MIN: usize = 10;

/// Kinds of outside evidence that are a measurement on the board itself. Anything else is a document.
pub const MEASURED_KINDS: [&str; 4] = ["edge-micrograph", "cross-section", "xray", "ct"];

fn st<'a>(v: &'a Value, k: &str) -> &'a str { v[k].as_str().unwrap_or("") }
fn xy(v: &Value) -> Option<(f64, f64)> { Some((v.get(0)?.as_f64()?, v.get(1)?.as_f64()?)) }

/// What the photos say about the layer count: counts, and the minimum that follows.
pub fn photo_evidence(flow: &Path) -> Value {
    let copper = items::read_json(&flow.join("copper.json")).unwrap_or(json!({}));
    let on = |layer: &str| copper["zones"].as_array().into_iter().flatten().chain(copper["traces"].as_array().into_iter().flatten()).any(|z| st(z, "layer") == layer);
    let faces = usize::from(on("F.Cu")) + usize::from(on("B.Cu"));
    let vias: Vec<&Value> = copper["vias"].as_array().into_iter().flatten().collect();
    let inner = vias.iter().filter(|v| st(v, "kind") == "inner").count();
    let both = vias.iter().filter(|v| st(v, "seen") == "both-faces").count();
    let hidden = vias.iter().filter(|v| st(v, "seen").contains("hidden")).count();
    // every candidate either detector looked at, kept or not, by face
    let mut cands: Vec<(String, f64, f64)> = vec![];
    for f in ["detect/dome-candidates.json", "detect/pit-candidates.json"] {
        if let Ok(d) = items::read_json(&flow.join(f)) { for c in d["candidates"].as_array().into_iter().flatten() { if let Some((x, y)) = xy(&c["pos_mm"]) { cands.push((st(c, "side").to_string(), x, y)); } } }
    }
    let have_cands = !cands.is_empty();
    let (mut one, mut clear) = (0usize, 0usize);
    for v in vias.iter().filter(|v| matches!(st(v, "seen"), "top" | "bottom")) {
        one += 1;
        let Some((x, y)) = xy(&v["pos_mm"]) else { continue };
        let other = if st(v, "seen") == "top" { "bottom" } else { "top" };
        if have_cands && !cands.iter().any(|c| c.0 == other && (c.1 - x).abs() <= OTHER_FACE_CLEAR_MM && (c.2 - y).abs() <= OTHER_FACE_CLEAR_MM) { clear += 1; }
    }
    let (layers, tag, why) = if inner > 0 {
        (4, "MEASURED", format!("{inner} via(s) are ringed on both faces with no copper leaving on either, so they continue on an inner layer: at least 4 layers"))
    } else if clear >= ONE_FACE_MIN && both >= BOTH_FACES_MIN {
        (4, "INFERRED", format!("{clear} vias show on one face while the other face is in view there and shows no via candidate within {OTHER_FACE_CLEAR_MM} mm, and {both} other vias show on both faces, so the photos do show through vias: a via that ends inside the board needs an inner layer, so more than 2 layers (4 in practice) is likely. Inferred, not proven: a via plugged or tented flat on one face reads the same"))
    } else if faces >= 2 {
        (2, "SEEN", "copper was read on both faces; nothing in the photos shows a via ending inside the board".to_string())
    } else {
        (1, "SEEN", "copper was read on one face only: the other face was not photographed or shows none".to_string())
    };
    json!({"facesWithCopper": faces, "vias": vias.len(), "viasOnBothFaces": both, "viasWithTheOtherFaceHidden": hidden, "viasOnOneFaceOnly": one, "viasOnOneFaceOtherFaceClear": clear, "innerVias": inner,
        "rule": {"otherFaceClearMm": OTHER_FACE_CLEAR_MM, "oneFaceMin": ONE_FACE_MIN, "bothFacesMin": BOTH_FACES_MIN},
        "minimum": {"layers": layers, "tag": tag, "why": why}})
}

/// An outside file, by hash. It must not be one of the run's own photos or products.
fn outside_file(flow: &Path, path: &str) -> Result<Value, String> {
    let p = items::resolve(flow, path).ok_or(format!("evidence file {path} is not there"))?;
    let full = std::fs::canonicalize(&p).map_err(|e| e.to_string())?;
    for d in ["photos", "board", "evidence", "audits", "detect"] {
        if let Ok(x) = std::fs::canonicalize(flow.join(d)) { if full.starts_with(x) { return Err(format!("{path} is this run's own {d}: outside evidence means an edge micrograph, a cross-section, an X-ray, a CT scan or a document")); } }
    }
    Ok(json!({"file": full.display().to_string(), "sha256": crate::sha256(&full)}))
}

/// What `layers state` was asked: a count with where it comes from, or nothing.
pub struct Ask<'a> { pub count: Option<usize>, pub source: Option<&'a str>, pub evidence: Option<&'a str>, pub kind: Option<&'a str> }

/// Write layers.json as a statement. With no count asked, a count the file already holds from outside evidence is kept; a
/// count that came from routing is not, and the routing record moves under `routing`.
pub fn state(flow: &Path, ask: &Ask) -> Result<(Value, Vec<String>), String> {
    let old = items::read_json(&flow.join("layers.json")).unwrap_or(json!({}));
    let ev = photo_evidence(flow);
    let min = ev["minimum"].clone();
    let min_n = min["layers"].as_u64().unwrap_or(2) as usize;
    let mut hints = vec![format!("photos: {}", st(&min, "why"))];
    // what is stated: asked now, or kept from an earlier statement that rests on outside evidence
    let kept = old["proposed"].as_u64().filter(|_| matches!(st(&old, "tag"), "DATASHEET" | "MEASURED") && (!st(&old, "source").is_empty() || old["evidenceFile"].is_object()));
    let mut doc = json!({"method": "stated", "tests": ["T-layers"], "planes": [], "photoEvidence": ev, "minimum": min});
    match (ask.count, kept) {
        (Some(0), _) | (None, None) => {
            doc["proposed"] = Value::Null; doc["tag"] = json!("UNKNOWN");
            doc["why"] = json!(format!("The layer count is unknown: photos show two faces and never an inner layer, and no outside evidence was given. From the photos: {}. T-layers settles it: count the copper layers on the board edge under a microscope, or X-ray the board, then `layers state --count <n> --evidence <file> --kind edge-micrograph`", st(&min, "why")));
            hints.push("layer count UNKNOWN, with test T-layers. Outside evidence states it: `layers state --count <n> --source \"<document>\"` or `--evidence <file> --kind edge-micrograph|cross-section|xray|ct`".into());
        }
        (Some(n), _) => {
            if n % 2 == 1 && n > 1 { hints.push(format!("{n} is an odd layer count: unusual, check the source")); }
            if n < min_n && st(&min, "tag") != "INFERRED" { return Err(format!("{n} layers stated, but the photos prove at least {min_n}: {}", st(&min, "why"))); }
            if n < min_n { hints.push(format!("{n} layers stated, but the photos suggest at least {min_n}: {}", st(&min, "why"))); }
            let file = match ask.evidence { Some(p) => Some(outside_file(flow, p)?), None => None };
            let kind = ask.kind.unwrap_or(if file.is_some() { "" } else { "document" });
            let measured = MEASURED_KINDS.contains(&kind);
            if measured && file.is_none() { return Err(format!("--kind {kind} is a measurement on the board: attach it with --evidence <file>")); }
            if file.is_some() && kind.is_empty() { return Err(format!("--evidence needs --kind: {} (a measurement on the board) or document", MEASURED_KINDS.join(", "))); }
            let source = ask.source.unwrap_or("").trim();
            if !measured && source.is_empty() { return Err("a count from a document needs --source \"<document, where in it, when fetched>\"".into()); }
            doc["proposed"] = json!(n); doc["tag"] = json!(if measured { "MEASURED" } else { "DATASHEET" });
            doc["source"] = json!(if source.is_empty() { format!("{kind} of this board") } else { source.to_string() });
            if let Some(f) = &file { doc["evidenceFile"] = json!({"file": f["file"], "sha256": f["sha256"], "kind": kind}); doc["evidence"] = json!([f["file"]]); }
            doc["why"] = json!(format!("{n} layers, stated from outside evidence ({}), not from the photos and not from routing. From the photos: {}", st(&doc, "source"), st(&min, "why")));
            hints.push(format!("{n} layers stated, tag {}. The layers score stays capped until an independent verdict with that evidence attached: `audit draw --category layers`, then `audit verdict --category layers ... --evidence <file> --kind <kind>`", st(&doc, "tag")));
        }
        (None, Some(n)) => {
            for k in ["proposed", "tag", "source", "evidenceFile", "evidence"] { if !old[k].is_null() { doc[k] = old[k].clone(); } }
            doc["why"] = json!(format!("{n} layers, stated from outside evidence ({}), not from the photos and not from routing. From the photos: {}", st(&old, "source"), st(&min, "why")));
            hints.push(format!("kept the count already stated from outside evidence: {n} layers ({}). `layers state --count 0` withdraws it", st(&old, "tag")));
        }
    }
    // what an earlier optional routing recorded is kept, out of the statement's way
    let had = old["candidates"].as_array().map(|a| !a.is_empty()).unwrap_or(false);
    doc["routing"] = if had { json!({"ran": true, "note": "an earlier optional `connect route`; its stackups are kept here and no longer set the count", "candidates": old["candidates"], "trail": old["trail"], "escalation": old["escalation"], "said": if old["routingSaid"].is_null() { old["why"].clone() } else { old["routingSaid"].clone() }}) }
        else if old["routing"]["ran"] == json!(true) { old["routing"].clone() }
        else { json!({"ran": false, "note": "hidden connections are left unrouted (unrouted.json): `connect route` is optional and proves no layer count"}) };
    std::fs::write(flow.join("layers.json"), serde_json::to_string_pretty(&doc).map_err(|e| e.to_string())? + "\n").map_err(|e| e.to_string())?;
    Ok((doc, hints))
}

/// Whether layers.json is a statement that still matches the photos' evidence.
pub fn current(flow: &Path) -> bool {
    let Ok(d) = items::read_json(&flow.join("layers.json")) else { return false };
    d["method"] == json!("stated") && d["photoEvidence"] == photo_evidence(flow)
}

/// `check layers` on a statement: honest about what is known, and nothing typed past it.
pub fn check(flow: &Path, d: &Value, pcb: Option<&Path>) -> Outcome {
    let mut errs = vec![];
    for it in items::items_of("layers.json", d) { items::validate(flow, &it, &mut errs); }
    let ev = photo_evidence(flow);
    if d["photoEvidence"] != ev { errs.push("the photos' evidence in layers.json is not what copper.json gives now: run `layers state` again (it keeps a count stated from outside evidence)".into()); }
    let min_n = ev["minimum"]["layers"].as_u64().unwrap_or(2);
    if st(d, "why").is_empty() { errs.push("`why` missing: run `layers state`".into()); }
    match d["proposed"].as_u64() {
        None => {
            if st(d, "tag") != "UNKNOWN" { errs.push(format!("no layer count is stated, so the tag must be UNKNOWN, not {:?}: run `layers state`", st(d, "tag"))); }
            if d["tests"].as_array().map(|a| a.is_empty()).unwrap_or(true) { errs.push("an unknown layer count needs its test (T-layers): run `layers state`".into()); }
        }
        Some(n) => {
            match st(d, "tag") {
                "DATASHEET" => if st(d, "source").is_empty() { errs.push(format!("{n} layers tagged DATASHEET with no source: `layers state --count {n} --source \"<document>\"`")); },
                "MEASURED" => match d["evidenceFile"]["file"].as_str() {
                    Some(f) if crate::sha256(Path::new(f)) == st(&d["evidenceFile"], "sha256") => {}
                    Some(f) => errs.push(format!("the evidence file {f} is missing or changed since it was stated: `layers state --count {n} --evidence <file> --kind <kind>` again")),
                    None => errs.push(format!("{n} layers tagged MEASURED with no evidence file: `layers state --count {n} --evidence <file> --kind edge-micrograph|cross-section|xray|ct`")),
                },
                t => errs.push(format!("{n} layers stated with tag {t:?}: without routing a count comes from outside evidence only (DATASHEET or MEASURED). `layers state --count {n} --source ...`, or `layers state --count 0` for UNKNOWN")),
            }
            if n < min_n && st(&ev["minimum"], "tag") != "INFERRED" { errs.push(format!("{n} layers stated, but the photos prove at least {min_n}: {}", st(&ev["minimum"], "why"))); }
            if let Some(p) = pcb {
                match std::fs::read_to_string(p) {
                    Ok(t) => { let have = crate::kicad_in::copper_layers(&t); if have as u64 != n { errs.push(format!("{} has {have} copper layers, {n} stated: run `build` (it takes the stated count) and check again", p.display())); } }
                    Err(e) => errs.push(format!("{}: {e}", p.display())),
                }
            }
        }
    }
    if !errs.is_empty() { return Err(errs); }
    let nets: Vec<Value> = items::read_json(&flow.join("nets.json")).ok().and_then(|n| n["nets"].as_array().cloned()).unwrap_or_default();
    let hidden = crate::unrouted::count(&nets);
    let mut hints = vec![format!("photos: {}", st(&ev["minimum"], "why"))];
    hints.push(match d["proposed"].as_u64() { Some(n) => format!("{n} layers stated ({}): {}", st(d, "tag"), st(d, "source")), None => "layer count UNKNOWN, settled by T-layers".to_string() });
    hints.push(format!("{hidden} hidden connection(s) left unrouted as ratsnest, each with its evidence tag: unrouted.json. `connect route` is optional"));
    Ok(Pass { summary: json!({"proposed": d["proposed"], "tag": d["tag"], "minimum": ev["minimum"], "routed": false, "hiddenConnections": hidden}), hints, files: vec!["layers.json".into(), "nets.json".into(), "copper.json".into()] })
}

#[cfg(test)]
mod tests {
    use super::*;

    fn dir(name: &str) -> std::path::PathBuf {
        let d = std::env::temp_dir().join(format!("layerstate-{name}-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&d);
        std::fs::create_dir_all(d.join("detect")).unwrap();
        d
    }
    fn copper(d: &Path, vias: Value) {
        std::fs::write(d.join("copper.json"), json!({"outline": {"points_mm": [[100, 100], [140, 100], [140, 140]], "tag": "MEASURED", "evidence": ["copper.json"]},
            "zones": [{"id": "a", "layer": "F.Cu", "net": "GND"}, {"id": "b", "layer": "B.Cu", "net": "GND"}], "traces": [], "vias": vias}).to_string()).unwrap();
        std::fs::write(d.join("nets.json"), json!({"nets": [{"name": "GND", "pins": ["R1.1", "R2.1"], "copperGroups": []}]}).to_string()).unwrap();
    }

    #[test]
    fn with_nothing_given_the_count_is_unknown_with_its_test_and_the_check_passes() {
        let d = dir("unknown");
        copper(&d, json!([{"id": "v1", "pos_mm": [110, 110], "seen": "both-faces", "kind": "seen"}]));
        let (doc, _) = state(&d, &Ask { count: None, source: None, evidence: None, kind: None }).unwrap();
        assert!(doc["proposed"].is_null());
        assert_eq!(doc["tag"], "UNKNOWN");
        assert_eq!(doc["tests"], json!(["T-layers"]));
        assert_eq!(doc["minimum"]["layers"], 2);
        assert_eq!(doc["minimum"]["tag"], "SEEN");
        let pass = check(&d, &doc, None).unwrap();
        assert_eq!(pass.summary["routed"], json!(false));
        assert_eq!(pass.summary["hiddenConnections"], json!(1));
        // the layers item is open, so a test is generated for it
        assert!(items::items_of("layers.json", &doc).iter().any(|i| i.id == "layers" && i.open()));
        std::fs::remove_dir_all(&d).ok();
    }

    #[test]
    fn a_count_needs_a_document_or_a_measurement_and_cannot_go_under_what_the_photos_prove() {
        let d = dir("count");
        copper(&d, json!([{"id": "v1", "pos_mm": [110, 110], "seen": "both-faces", "kind": "inner"}]));
        assert!(state(&d, &Ask { count: Some(6), source: None, evidence: None, kind: None }).is_err());
        assert!(state(&d, &Ask { count: Some(6), source: None, evidence: None, kind: Some("xray") }).is_err());
        assert!(state(&d, &Ask { count: Some(2), source: Some("maker's page"), evidence: None, kind: None }).err().unwrap().contains("at least 4"));
        let (doc, _) = state(&d, &Ask { count: Some(6), source: Some("maker's page, fetched today"), evidence: None, kind: None }).unwrap();
        assert_eq!(doc["tag"], "DATASHEET");
        assert_eq!(doc["minimum"]["layers"], 4);
        assert!(check(&d, &doc, None).is_ok());
        // stated again with nothing asked, the count from outside evidence is kept; `--count 0` withdraws it
        let (again, _) = state(&d, &Ask { count: None, source: None, evidence: None, kind: None }).unwrap();
        assert_eq!(again["proposed"], 6);
        let (gone, _) = state(&d, &Ask { count: Some(0), source: None, evidence: None, kind: None }).unwrap();
        assert!(gone["proposed"].is_null());
        // a micrograph of the board is a measurement, held by hash
        std::fs::write(d.join("edge.jpg"), "x").unwrap();
        let (m, _) = state(&d, &Ask { count: Some(6), source: None, evidence: Some("edge.jpg"), kind: Some("edge-micrograph") }).unwrap();
        assert_eq!(m["tag"], "MEASURED");
        assert!(check(&d, &m, None).is_ok());
        std::fs::write(d.join("edge.jpg"), "changed").unwrap();
        assert!(check(&d, &m, None).is_err());
        std::fs::remove_dir_all(&d).ok();
    }

    #[test]
    fn a_count_from_routing_is_not_kept_and_a_typed_one_fails_the_check() {
        let d = dir("routed");
        copper(&d, json!([]));
        std::fs::write(d.join("layers.json"), json!({"proposed": 6, "tag": "INFERRED", "why": "routing", "tests": ["T-layers"], "candidates": [{"layers": 6, "open": 3, "run": "routes/L6"}]}).to_string()).unwrap();
        let (doc, _) = state(&d, &Ask { count: None, source: None, evidence: None, kind: None }).unwrap();
        assert!(doc["proposed"].is_null());
        assert_eq!(doc["routing"]["ran"], json!(true));
        assert_eq!(doc["routing"]["candidates"][0]["layers"], 6);
        assert!(doc["candidates"].is_null());
        let mut typed = doc.clone(); typed["proposed"] = json!(4); typed["tag"] = json!("INFERRED");
        assert!(check(&d, &typed, None).is_err());
        // and evidence that changed under the statement asks for a new one
        copper(&d, json!([{"id": "v1", "pos_mm": [110, 110], "seen": "both-faces", "kind": "inner"}]));
        assert!(!current(&d));
        assert!(check(&d, &doc, None).err().unwrap()[0].contains("layers state"));
        std::fs::remove_dir_all(&d).ok();
    }

    #[test]
    fn vias_that_end_on_one_face_suggest_inner_layers_only_when_the_other_face_is_clear() {
        let d = dir("oneface");
        let mut vias: Vec<Value> = (0..12).map(|i| json!({"id": format!("b{i}"), "pos_mm": [101.0 + i as f64, 101.0], "seen": "both-faces", "kind": "seen"})).collect();
        vias.extend((0..12).map(|i| json!({"id": format!("t{i}"), "pos_mm": [101.0 + i as f64, 110.0], "seen": "top", "kind": "seen"})));
        vias.extend((0..5).map(|i| json!({"id": format!("h{i}"), "pos_mm": [101.0 + i as f64, 120.0], "seen": "bottom, top hidden by a part", "kind": "seen"})));
        copper(&d, json!(vias));
        // no candidate list at all: nothing can be called clear
        assert_eq!(photo_evidence(&d)["minimum"]["layers"], 2);
        // a rejected candidate on the bottom under every one-face via: the other face is not clear
        let under: Vec<Value> = (0..12).map(|i| json!({"pos_mm": [101.1 + i as f64, 110.1], "side": "bottom"})).collect();
        std::fs::write(d.join("detect/dome-candidates.json"), json!({"candidates": under}).to_string()).unwrap();
        assert_eq!(photo_evidence(&d)["viasOnOneFaceOtherFaceClear"], 0);
        assert_eq!(photo_evidence(&d)["minimum"]["layers"], 2);
        // candidates elsewhere only: the bottom is clear under them, and the minimum is inferred, never measured
        let away: Vec<Value> = (0..12).map(|i| json!({"pos_mm": [101.0 + i as f64, 130.0], "side": "bottom"})).collect();
        std::fs::write(d.join("detect/dome-candidates.json"), json!({"candidates": away}).to_string()).unwrap();
        let ev = photo_evidence(&d);
        assert_eq!(ev["viasOnOneFaceOtherFaceClear"], 12);
        assert_eq!(ev["viasWithTheOtherFaceHidden"], 5);
        assert_eq!(ev["minimum"]["layers"], 4);
        assert_eq!(ev["minimum"]["tag"], "INFERRED");
        std::fs::remove_dir_all(&d).ok();
    }
}