AI Flow: Reverse Engineering

//! The connections nobody can see, left as they are: unrouted. The default flow does not invent inner-layer
//! routing for them. Each one stays a ratsnest line in the KiCad copy and a row in `unrouted.json`, with the
//! evidence tag of what says the two ends belong together: `DATASHEET` (a document names both pins the same),
//! `INFERRED` (the circuit suggests it) or `MEASURED` (a bench probe recorded by `bench ingest`).
//!
//! A net's pins fall into pieces: each group of pins that seen copper joins (`copperGroups`, written by
//! `connect attribute`), and each pin no copper is seen to reach, alone. A net of g pieces has g - 1 hidden
//! connections, each from the largest piece to one of the others. This reads nets.json only, so it is the same
//! whether or not the optional router ever ran.
use serde_json::{json, Value};
use std::collections::{BTreeMap, HashSet};
use std::path::Path;

/// One hidden connection: the piece `to` is not seen joined to the net's largest piece `from`.
#[derive(Clone, Debug)]
pub struct Hidden { pub id: String, pub net: String, pub from: Vec<String>, pub to: Vec<String>, pub tag: String, pub source: String, pub tests: Vec<String> }

fn strs(v: &Value) -> Vec<String> { v.as_array().into_iter().flatten().filter_map(|x| x.as_str().map(str::to_string)).collect() }

/// The pieces of one net, largest first; ties and the rest in the order of their first pin, so the list does not depend on file order.
pub fn pieces(net: &Value) -> Vec<Vec<String>> {
    let pins = strs(&net["pins"]);
    let of: HashSet<&str> = pins.iter().map(String::as_str).collect();
    let mut used: HashSet<String> = HashSet::new();
    let mut out: Vec<Vec<String>> = vec![];
    for g in net["copperGroups"].as_array().into_iter().flatten() {
        let mut m: Vec<String> = strs(g).into_iter().filter(|p| of.contains(p.as_str()) && used.insert(p.clone())).collect();
        m.sort();
        if !m.is_empty() { out.push(m); }
    }
    for p in &pins { if used.insert(p.clone()) { out.push(vec![p.clone()]); } }
    out.sort_by(|a, b| b.len().cmp(&a.len()).then(a[0].cmp(&b[0])));
    out
}

/// What says the hidden ends of this net belong together: a document the net cites, or an inference.
pub fn net_tag(net: &Value) -> &'static str {
    let cited = |k: &str| { let t = net[k].as_str().unwrap_or("").trim().to_lowercase(); !(t.is_empty() || t.starts_with("none") || t.starts_with("infer") || matches!(t.as_str(), "measured" | "router" | "copper" | "silkscreen" | "circuit" | "datasheet")) };
    if net["tag"] == json!("DATASHEET") || cited("source") { "DATASHEET" } else { "INFERRED" }
}

/// Every hidden connection of the run, from nets.json as it stands.
pub fn connections(nets: &[Value]) -> Vec<Hidden> {
    let mut out = vec![];
    for n in nets {
        let name = n["name"].as_str().unwrap_or("").to_string();
        let ps = pieces(n);
        let Some(root) = ps.first().cloned() else { continue };
        for piece in ps.iter().skip(1) {
            out.push(Hidden { id: format!("hc:{name}:{}", piece[0]), net: name.clone(), from: root.clone(), to: piece.clone(), tag: net_tag(n).to_string(), source: n["source"].as_str().unwrap_or("").to_string(), tests: strs(&n["tests"]) });
        }
    }
    out
}

/// How many hidden connections nets.json holds: what a router would have to add, and what a bench probe would settle.
pub fn count(nets: &[Value]) -> usize { nets.iter().map(|n| pieces(n).len().saturating_sub(1)).sum() }

/// The hidden connections a recorded bench probe found joined, by id (`bench/measured.json`, written by `bench ingest`).
pub fn measured(flow: &Path) -> BTreeMap<String, Value> {
    let doc = crate::items::read_json(&flow.join(crate::bench::MEASURED)).unwrap_or(json!({}));
    doc["connections"].as_object().into_iter().flatten().filter(|(_, v)| v["result"] == json!("joined")).map(|(k, v)| (k.clone(), v.clone())).collect()
}

/// Of the hidden connections that exist now, the ones a bench probe found joined. A record for a connection that is no longer
/// in nets.json (its pins were moved) counts for nothing.
pub fn measured_now(flow: &Path, nets: &[Value]) -> Vec<Hidden> {
    let m = measured(flow);
    connections(nets).into_iter().filter(|h| m.get(&h.id).map(|v| strs(&v["to"]) == h.to).unwrap_or(false)).collect()
}

/// The pads a recorded probe touched and found joined, for connections that still stand: each is proven on its net by measurement.
pub fn probed_pins(flow: &Path, nets: &[Value]) -> HashSet<String> {
    let m = measured(flow);
    measured_now(flow, nets).iter().filter_map(|h| m.get(&h.id)).flat_map(|v| [v["a"].as_str().unwrap_or("").to_string(), v["b"].as_str().unwrap_or("").to_string()]).filter(|p| !p.is_empty()).collect()
}

/// `unrouted.json`: the list, for reading. Nothing reads it back: scores and the bench plan compute it again.
pub fn write(flow: &Path) -> Value {
    let nets: Vec<Value> = crate::items::read_json(&flow.join("nets.json")).ok().and_then(|d| d["nets"].as_array().cloned()).unwrap_or_default();
    let m = measured(flow);
    let mut by: BTreeMap<String, usize> = BTreeMap::new();
    let rows: Vec<Value> = connections(&nets).iter().map(|h| {
        let bench = m.get(&h.id).filter(|v| strs(&v["to"]) == h.to);
        let tag = if bench.is_some() { "MEASURED" } else { h.tag.as_str() };
        *by.entry(tag.to_string()).or_default() += 1;
        json!({"id": h.id, "net": h.net, "from": h.from, "to": h.to, "tag": tag, "source": h.source, "tests": h.tests, "bench": bench})
    }).collect();
    let doc = json!({"derived": "written by the program for reading; nothing reads it back", "what": "connections the photos do not show, left unrouted: each is a ratsnest line in the KiCad copy between the piece `from` (the net's largest group of pins joined by seen copper) and the piece `to`",
        "count": rows.len(), "byTag": by, "connections": rows});
    let _ = std::fs::write(flow.join("unrouted.json"), serde_json::to_string_pretty(&doc).unwrap_or_default() + "\n");
    doc
}

/// The evidence class of a whole net, for the KiCad copy: what its ratsnest lines rest on. A net of one pin has nothing to join and stays in Default.
pub fn net_class(net: &Value, joined: &HashSet<String>) -> &'static str {
    let name = net["name"].as_str().unwrap_or("");
    let ps = pieces(net);
    if ps.len() <= 1 { return if ps.first().is_some_and(|p| p.len() >= 2) { "SEEN" } else { "Default" }; }
    if ps.iter().skip(1).all(|p| joined.contains(&format!("hc:{name}:{}", p[0]))) { return "BENCH_MEASURED"; }
    if net_tag(net) == "DATASHEET" { "HIDDEN_DATASHEET" } else { "HIDDEN_INFERRED" }
}

/// The classes the KiCad project carries beside Default, with the colour of their ratsnest and copper.
pub const CLASSES: [(&str, &str, &str); 4] = [
    ("SEEN", "rgba(0, 170, 80, 1.000)", "every pin of the net is joined by copper the photos show: no ratsnest line is expected"),
    ("HIDDEN_DATASHEET", "rgba(0, 120, 255, 1.000)", "ratsnest lines of this net are connections no photo shows, named by a document the net cites"),
    ("HIDDEN_INFERRED", "rgba(255, 140, 0, 1.000)", "ratsnest lines of this net are connections no photo shows, inferred from the circuit: each needs its test"),
    ("BENCH_MEASURED", "rgba(170, 0, 200, 1.000)", "every hidden connection of this net was found joined by a recorded bench probe"),
];

/// The KiCad project text with one net class per kind of evidence and every net assigned to its class, so a ratsnest line
/// shows in KiCad what it rests on. The classes repeat Default's sizes, so no rule changes.
pub fn with_evidence_classes(pro: &str, flow: &Path) -> String {
    let Ok(mut doc) = serde_json::from_str::<Value>(pro) else { return pro.to_string() };
    let nets: Vec<Value> = crate::items::read_json(&flow.join("nets.json")).ok().and_then(|d| d["nets"].as_array().cloned()).unwrap_or_default();
    if nets.is_empty() { return pro.to_string(); }
    let joined: HashSet<String> = measured_now(flow, &nets).into_iter().map(|h| h.id).collect();
    let default = doc["net_settings"]["classes"][0].clone();
    let Some(classes) = doc["net_settings"]["classes"].as_array_mut() else { return pro.to_string() };
    for (name, colour, what) in CLASSES { let mut c = default.clone(); c["name"] = json!(name); c["pcb_color"] = json!(colour); c["description"] = json!(what); classes.push(c); }
    // a pattern is a wildcard match in KiCad: the characters it reads as wildcards are escaped by matching any one character
    let pattern = |n: &str| n.chars().map(|c| if matches!(c, '*' | '?' | '[' | ']' | '{' | '}' | '\\') { '?' } else { c }).collect::<String>();
    let pats: Vec<Value> = nets.iter().filter_map(|n| { let name = n["name"].as_str()?; let class = net_class(n, &joined); (class != "Default").then(|| json!({"netclass": class, "pattern": pattern(&name.replace('"', "'"))})) }).collect();
    doc["net_settings"]["netclass_patterns"] = json!(pats);
    serde_json::to_string_pretty(&doc).unwrap_or_else(|_| pro.to_string()) + "\n"
}

#[cfg(test)]
mod tests {
    use super::*;

    fn net(name: &str, pins: &[&str], groups: &[&[&str]]) -> Value { json!({"name": name, "pins": pins, "copperGroups": groups, "tag": "INFERRED", "tests": [format!("T-net-{name}")]}) }

    #[test]
    fn a_net_of_three_pieces_has_two_hidden_connections_from_its_largest() {
        let n = net("VCC", &["U1.1", "C1.1", "C2.1", "J1.3"], &[&["C1.1", "C2.1"], &["PHOTO1.M3"]]);
        let ps = pieces(&n);
        assert_eq!(ps, vec![vec!["C1.1".to_string(), "C2.1".to_string()], vec!["J1.3".to_string()], vec!["U1.1".to_string()]]);
        let h = connections(&[n.clone()]);
        assert_eq!(h.len(), 2);
        assert_eq!(count(&[n]), 2);
        assert_eq!(h[0].id, "hc:VCC:J1.3");
        assert_eq!(h[0].from, vec!["C1.1", "C2.1"]);
        assert_eq!(h[0].tag, "INFERRED");
    }

    #[test]
    fn a_net_joined_by_seen_copper_has_none_and_a_cited_net_is_tagged_by_its_document() {
        assert!(connections(&[net("A", &["R1.1", "R2.1"], &[&["R1.1", "R2.1"]])]).is_empty());
        let mut n = net("+5V", &["J1.77", "J1.79"], &[]);
        n["source"] = json!("maker's datasheet, Table 4");
        let h = connections(&[n]);
        assert_eq!(h.len(), 1);
        assert_eq!(h[0].tag, "DATASHEET");
    }

    #[test]
    fn the_kicad_project_gets_a_class_per_kind_of_evidence_and_keeps_the_rules() {
        let d = std::env::temp_dir().join(format!("unrouted-test-{}", std::process::id()));
        std::fs::create_dir_all(d.join("bench")).unwrap();
        let mut cited = net("+5V", &["J1.77", "J1.79"], &[]); cited["source"] = json!("datasheet Table 4");
        std::fs::write(d.join("nets.json"), json!({"nets": [net("A", &["R1.1", "R2.1"], &[&["R1.1", "R2.1"]]), net("B*", &["R3.1", "R4.1"], &[]), cited, net("M", &["R5.1", "R6.1"], &[]), net("ONE", &["J1.5"], &[])]}).to_string()).unwrap();
        std::fs::write(d.join(crate::bench::MEASURED), json!({"connections": {"hc:M:R6.1": {"result": "joined", "to": ["R6.1"]}}}).to_string()).unwrap();
        let pro = crate::rules::kicad_pro(&crate::rules::profile("kicad-default").unwrap(), "x");
        let out: Value = serde_json::from_str(&with_evidence_classes(&pro, &d)).unwrap();
        let classes = out["net_settings"]["classes"].as_array().unwrap();
        assert_eq!(classes.len(), 5);
        for c in classes { assert_eq!(c["clearance"], json!(0.2)); assert_eq!(c["track_width"], json!(0.2)); }
        let class_of = |n: &str| out["net_settings"]["netclass_patterns"].as_array().unwrap().iter().find(|p| p["pattern"] == json!(n)).map(|p| p["netclass"].as_str().unwrap().to_string());
        assert_eq!(class_of("A").as_deref(), Some("SEEN"));
        assert_eq!(class_of("B?").as_deref(), Some("HIDDEN_INFERRED"));
        assert_eq!(class_of("+5V").as_deref(), Some("HIDDEN_DATASHEET"));
        assert_eq!(class_of("M").as_deref(), Some("BENCH_MEASURED"));
        assert_eq!(class_of("ONE"), None, "a net of one pin has nothing to join: it stays in Default");
        assert_eq!(out["board"]["design_settings"]["rules"]["min_clearance"], json!(0.2));
        let doc = write(&d);
        assert_eq!(doc["count"], json!(3));
        assert_eq!(doc["byTag"]["MEASURED"], json!(1));
        std::fs::remove_dir_all(&d).ok();
    }
}