AI Flow: Reverse Engineering

//! Datasheet pins: what a chip's public datasheet says of every pin, as a file the program can read.
//!
//! `reverse-flow/datasheets/<ref>.pins.json` is written by the AI for each identified chip, from the datasheet it fetched
//! during the run, and cites the document by name, revision and page:
//!
//! ```json
//! {"part": "U2", "mpn": "DRV8323SRTAR",
//!  "document": {"name": "TI DRV832x 6 to 60-V Three-Phase Smart Gate Driver", "number": "SLVSDJ3", "revision": "D, March 2022", "page": 8, "file": "datasheets/ti-drv8323-slvsdj3.pdf"},
//!  "pins": [{"pad": "3", "name": "VCP", "type": "power_out", "expects": [{"kind": "capacitor_to_pin", "pin": "VM", "note": "1 uF, 25 V"}]}, ...]}
//! ```
//!
//! or, when there is no public datasheet or the marking does not settle the exact part, `{"part": "U5", "available": false,
//! "reason": "..."}`. A part whose marking was not read gets no datasheet witness, whatever file is there: identities are not guessed.
//!
//! What the program does with it:
//! - `datasheet apply` writes each part's `pins` (pad to name and KiCad pin type) into parts.json, so the schematic draws the
//!   chip with its real pin names. `pinsSource` holds the citation.
//! - attribution names a conductor that no net of the pin map reached from the named pin that sits on it (`Names::name_for`),
//!   and `stamp` tags that name DATASHEET with the document cited. The join stays exactly as the copper shows it.
//! - `auto` stops once with "AUTO NEEDS AI: datasheet pins" while an identified chip has neither file nor a "none available".
use crate::{items, kicad_out, sch};
use serde_json::{json, Value};
use std::collections::BTreeMap;
use std::path::Path;

pub const DIR: &str = "datasheets";
/// The electrical types a pins file may state.
pub const TYPES: [&str; 9] = ["power_in", "power_out", "ground", "input", "output", "bidirectional", "open_drain", "passive", "no_connect"];
/// What a pin may be said to expect around it. The witness (`connect witness`) checks each against the copper and the parts.
pub const EXPECTS: [&str; 11] = ["capacitor_to_ground", "capacitor_to_pin", "tied_to_pin", "tied_to_ground", "resistor_to_pin", "resistor_to_ground", "pull_up", "crystal", "inductor", "not_connected", "other"];

/// The KiCad pin type the schematic generator takes for one of ours.
pub fn kicad_type(t: &str) -> &'static str {
    match t { "power_in" | "ground" => "power_in", "power_out" => "power_out", "input" => "input", "output" => "output", "bidirectional" => "bidirectional", "open_drain" => "open_collector", _ => "passive" }
}

#[derive(Clone, Debug)]
pub struct Pin { pub pad: String, pub name: String, pub ty: String, pub net: Option<String>, pub expects: Vec<Value> }

#[derive(Clone, Debug)]
pub struct Sheet { pub part: String, pub file: String, pub cite: String, pub mpn: String, pub pins: Vec<Pin> }

impl Sheet {
    pub fn pin(&self, pad: &str) -> Option<&Pin> { self.pins.iter().find(|p| p.pad == pad) }
    /// Pads a pin name or a pad number stands for (a name may sit on several pads: GND, VDD).
    pub fn pads_named(&self, what: &str) -> Vec<String> {
        let by_name: Vec<String> = self.pins.iter().filter(|p| p.name.eq_ignore_ascii_case(what)).map(|p| p.pad.clone()).collect();
        if !by_name.is_empty() { return by_name; }
        self.pins.iter().filter(|p| p.pad == what).map(|p| p.pad.clone()).collect()
    }
}

/// A chip the stop names: an IC or a transistor of three pads or more whose marking was read.
#[derive(Clone, Debug)]
pub struct Chip { pub r: String, pub value: String, pub marking: String, pub footprint: String, pub pads: usize }

#[derive(Default)]
pub struct Status {
    /// identified chips with neither a pins file nor a "none available"
    pub missing: Vec<Chip>,
    /// "none available", with the reason given
    pub declined: Vec<(String, String)>,
    /// valid sheets of identified parts, by reference
    pub sheets: BTreeMap<String, Sheet>,
    /// files that do not pass, with what is wrong
    pub bad: Vec<(String, Vec<String>)>,
    /// files for parts whose marking was not read: no witness
    pub unread: Vec<String>,
    pub chips: usize,
}

fn text(v: &Value) -> String { match v { Value::String(s) => s.trim().to_string(), Value::Number(n) => n.to_string(), _ => String::new() } }

/// The marking was read: the part's value (or part number) is tagged as seen, measured or from a document, never inferred or
/// unknown. With no field tag on either, a part number beside a marking that is not called unreadable counts as read.
pub fn identified(p: &Value) -> bool {
    let has = |k: &str| !p[k].as_str().unwrap_or("").trim().is_empty();
    if !has("mpn") && !has("value") { return false; }
    let ft = &p["fieldTags"];
    match ft["mpn"].as_str().or(ft["value"].as_str()).or(ft["marking"].as_str()) {
        Some(t) => matches!(t, "SEEN" | "MEASURED" | "DATASHEET"),
        None => { let m = p["marking"].as_str().unwrap_or("").to_lowercase(); has("mpn") && !m.trim().is_empty() && !["unreadable", "none", "not read", "?"].iter().any(|w| m.contains(w)) }
    }
}

fn pads_of(flow: &Path, p: &Value) -> Option<Vec<String>> {
    let fp = p["footprint"].as_str().unwrap_or("");
    if fp.is_empty() { return None; }
    kicad_out::footprint_text(&kicad_out::cache_dir(flow), fp, p).ok().map(|t| sch::pad_numbers(&t))
}

fn cite(doc: &Value) -> String {
    let mut s = text(&doc["name"]);
    let num = text(&doc["number"]); if !num.is_empty() { s += &format!(", {num}"); }
    s += &format!(", revision {}", text(&doc["revision"]));
    s += &format!(", page {}", text(&doc["page"]));
    s
}

/// One file read against its part: the sheet, or everything wrong with it. `pads`: the footprint's pads when they are known.
pub fn parse(v: &Value, r: &str, file: &str, pads: Option<&[String]>) -> Result<Sheet, Vec<String>> {
    let mut e = vec![];
    let doc = &v["document"];
    for k in ["name", "revision", "page"] { if text(&doc[k]).is_empty() { e.push(format!("{file}: document.{k} is missing: cite the datasheet by name, revision and the page of its pin table")); } }
    let mut pins: Vec<Pin> = vec![];
    let rows = v["pins"].as_array().cloned().unwrap_or_default();
    if rows.is_empty() { e.push(format!("{file}: no pins: list every pin as {{\"pad\", \"name\", \"type\"}}, or answer {{\"part\": \"{r}\", \"available\": false, \"reason\": \"...\"}}")); }
    for row in &rows {
        let (pad, name, ty) = (text(&row["pad"]), text(&row["name"]), text(&row["type"]));
        if pad.is_empty() { e.push(format!("{file}: a pin has no pad number: {row}")); continue; }
        if pins.iter().any(|p| p.pad == pad) { e.push(format!("{file}: pad {pad} is listed twice")); continue; }
        if name.is_empty() { e.push(format!("{file}: pad {pad} has no name")); }
        if !TYPES.contains(&ty.as_str()) { e.push(format!("{file}: pad {pad} ({name}): type {ty:?} is not one of {}", TYPES.join(", "))); }
        let expects: Vec<Value> = row["expects"].as_array().cloned().unwrap_or_default();
        for x in &expects { let k = text(&x["kind"]); if !EXPECTS.contains(&k.as_str()) { e.push(format!("{file}: pad {pad} ({name}): expects kind {k:?} is not one of {}", EXPECTS.join(", "))); } }
        let net = Some(text(&row["net"])).filter(|s| !s.is_empty());
        pins.push(Pin { pad, name, ty, net, expects });
    }
    let sheet = Sheet { part: r.to_string(), file: file.to_string(), cite: cite(doc), mpn: text(&v["mpn"]), pins };
    for p in &sheet.pins { for x in &p.expects { if matches!(text(&x["kind"]).as_str(), "capacitor_to_pin" | "tied_to_pin" | "resistor_to_pin") {
        let other = text(&x["pin"]);
        if other.is_empty() || sheet.pads_named(&other).is_empty() { e.push(format!("{file}: pad {} ({}): expects {} names pin {other:?}, which is no pin name or pad of this part", p.pad, p.name, text(&x["kind"]))); }
    } } }
    if let (Some(pads), false) = (pads, sheet.pins.is_empty()) {
        for p in &sheet.pins { if !pads.contains(&p.pad) { e.push(format!("{file}: pad {} ({}) is not on the footprint of {r} (its pads: {}): the footprint or the package variant is wrong", p.pad, p.name, span(pads))); } }
        let lacking: Vec<&String> = pads.iter().filter(|q| sheet.pin(q).is_none()).collect();
        if !lacking.is_empty() { e.push(format!("{file}: footprint pad(s) {} of {r} have no entry: every pad takes one (an exposed pad is what the datasheet says it is, usually ground; a pad the datasheet does not list means the footprint is not this package)", lacking.iter().map(|s| s.as_str()).collect::<Vec<_>>().join(", "))); }
    }
    if e.is_empty() { Ok(sheet) } else { Err(e) }
}

fn span(pads: &[String]) -> String { if pads.len() > 12 { format!("{} to {}, {} pads", pads[0], pads[pads.len() - 1], pads.len()) } else { pads.join(", ") } }

/// Every `*.pins.json`, by the reference it answers for: its `part` (and `refs` for identical parts), else the file's name.
fn files(flow: &Path) -> BTreeMap<String, (String, Value)> {
    let mut out = BTreeMap::new();
    let mut names: Vec<String> = std::fs::read_dir(flow.join(DIR)).into_iter().flatten().flatten().map(|e| e.file_name().to_string_lossy().to_string()).filter(|n| n.ends_with(".pins.json")).collect();
    names.sort();
    for n in names {
        let rel = format!("{DIR}/{n}");
        let v = items::read_json(&flow.join(&rel)).unwrap_or_else(|e| json!({"unreadable": e}));
        let mut refs: Vec<String> = v["refs"].as_array().into_iter().flatten().map(text).filter(|s| !s.is_empty()).collect();
        let part = text(&v["part"]);
        refs.push(if part.is_empty() { n.trim_end_matches(".pins.json").to_string() } else { part });
        for r in refs { out.entry(r).or_insert((rel.clone(), v.clone())); }
    }
    out
}

/// A part whose value, part number or function says nothing is fitted at its position (`"fitted": false` says it outright).
pub fn not_fitted(p: &Value) -> bool {
    if p["fitted"] == json!(false) || p["dnp"] == json!(true) { return true; }
    ["value", "mpn", "function"].iter().any(|k| {
        let t = p[*k].as_str().unwrap_or("").to_lowercase();
        ["not fitted", "unfitted", "not populated", "unpopulated", "not placed", "not mounted", "no part fitted", "empty footprint", "do not populate", "do not fit"].iter().any(|w| t.contains(w))
            || t.split(|c: char| !c.is_ascii_alphanumeric()).any(|w| matches!(w, "dnp" | "dnf" | "dni" | "nf"))
    })
}

/// Where the run stands: which chips still want an answer, which sheets hold, which files do not.
pub fn status(flow: &Path) -> Status {
    let mut st = Status::default();
    let doc = items::read_json(&flow.join("parts.json")).unwrap_or(Value::Null);
    let mut fs = files(flow);
    for p in doc["parts"].as_array().into_iter().flatten() {
        let r = p["ref"].as_str().unwrap_or("").to_string();
        let fp = p["footprint"].as_str().unwrap_or("");
        let read = identified(p);
        let kind = matches!(sch::kind_of(&r, fp), sch::Kind::Ic | sch::Kind::Q);
        // the footprint is read only for a part that is asked about or answered for: this runs on every step of `auto`
        let pads = if (kind && read && !not_fitted(p)) || fs.contains_key(&r) { pads_of(flow, p) } else { None };
        // a footprint with no pads at all (a body outline standing in for a package nobody could match) has nothing to name: not asked
        // a position with no part on it has no datasheet to read: not asked
        let chip = kind && !not_fitted(p) && pads.as_ref().map(|v| v.len() >= 3).unwrap_or(true);
        let pads = pads.filter(|v| !v.is_empty());
        if chip && read { st.chips += 1; }
        match fs.remove(&r) {
            None => { if chip && read { st.missing.push(Chip { r, value: text(&p["mpn"]).is_empty().then(|| text(&p["value"])).unwrap_or_else(|| text(&p["mpn"])), marking: text(&p["marking"]), footprint: fp.rsplit(':').next().unwrap_or(fp).to_string(), pads: pads.map(|v| v.len()).unwrap_or(0) }); } }
            Some((file, v)) => {
                if !v["unreadable"].is_null() { st.bad.push((r, vec![text(&v["unreadable"])])); continue; }
                if v["available"] == json!(false) {
                    let why = text(&v["reason"]);
                    if why.is_empty() { st.bad.push((r.clone(), vec![format!("{file}: \"available\": false needs a \"reason\" (no public datasheet found, the marking fits several parts, ...)")])); } else { st.declined.push((r, why)); }
                    continue;
                }
                if !read { st.unread.push(format!("{r} ({file}): its marking was not read (parts.json tags its value {}), so it gets no datasheet witness. Read the marking first, or remove the file", p["fieldTags"]["value"].as_str().unwrap_or("with no tag"))); continue; }
                match parse(&v, &r, &file, pads.as_deref()) { Ok(s) => { st.sheets.insert(r, s); } Err(e) => st.bad.push((r, e)) }
            }
        }
    }
    for (r, (file, _)) in fs { st.bad.push((r.clone(), vec![format!("{file}: {r} is not a part in parts.json")])); }
    st
}

/// The lines the stop prints: the format, then each chip that waits.
pub fn ask_lines(st: &Status) -> Vec<String> {
    let mut v = vec![];
    if !st.missing.is_empty() {
        v.push(format!("For each chip below, fetch its public datasheet and write reverse-flow/{DIR}/<ref>.pins.json: {{\"part\": \"U1\", \"mpn\": \"<orderable part>\", \"document\": {{\"name\", \"number\", \"revision\", \"page\": <page of the pin table>, \"url\" or \"file\"}}, \"pins\": [{{\"pad\": \"1\", \"name\": \"VDD\", \"type\": \"{}\", \"expects\": [{{\"kind\": \"capacitor_to_ground\", \"note\": \"100 nF\"}}]}}, ...]}}. Every pad of the footprint takes an entry, the exposed pad too. `expects` is optional and holds what the datasheet's typical application shows at the pin ({}; `pin` names the other pin for the *_to_pin kinds). docs/method.md has a worked example.", TYPES.join("|"), EXPECTS.join(", ")));
        v.push(format!("No public datasheet, or the marking does not settle the exact part and package: write {{\"part\": \"<ref>\", \"available\": false, \"reason\": \"<why>\"}} in the same file instead. Never write pins for a part you could not identify. Then run `auto` again."));
        for c in &st.missing { v.push(format!("{}: {} (marking \"{}\", {}, {} pads): reverse-flow/{DIR}/{}.pins.json is missing", c.r, c.value, c.marking, c.footprint, c.pads, c.r)); }
    }
    for (_, errs) in &st.bad { v.extend(errs.iter().take(12).cloned()); }
    v.extend(st.unread.iter().cloned());
    v
}

/// parts.json as it would stand with every valid sheet written into its part's `pins`. `None` when it stands so already.
fn applied_doc(flow: &Path, st: &Status) -> Option<Value> {
    let mut doc = items::read_json(&flow.join("parts.json")).ok()?;
    let mut changed = false;
    for p in doc["parts"].as_array_mut().into_iter().flatten() {
        let r = p["ref"].as_str().unwrap_or("").to_string();
        match st.sheets.get(&r) {
            Some(s) => {
                // several pads of one name that drive (two VCAP pins of one regulator) are one output inside the chip: the first keeps the
                // driving type and the rest are passive, as KiCad's own symbols do, so the ERC does not call the chip's own tie a conflict
                let mut drives: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
                let pins: serde_json::Map<String, Value> = s.pins.iter().map(|q| { let t = kicad_type(&q.ty); let again = matches!(t, "power_out" | "output") && !drives.insert(q.name.as_str()); (q.pad.clone(), json!({"name": q.name, "type": if again { "passive" } else { t }})) }).collect();
                let (pins, src) = (Value::Object(pins), json!(s.cite));
                if p["pins"] != pins || p["pinsSource"] != src { p["pins"] = pins; p["pinsSource"] = src; changed = true; }
            }
            // pins an earlier sheet gave go with the sheet; pins typed by hand (no pinsSource) are not ours to remove
            None => if p["pinsSource"].is_string() { if let Some(o) = p.as_object_mut() { o.remove("pins"); o.remove("pinsSource"); changed = true; } }
        }
    }
    changed.then_some(doc)
}

pub fn applied(flow: &Path, st: &Status) -> bool { applied_doc(flow, st).is_none() }

/// `datasheet apply`: every valid sheet's pin names and types into its part. Fails on a file that does not pass.
pub fn apply(flow: &Path) -> Result<Value, Vec<String>> {
    let st = status(flow);
    if !st.bad.is_empty() || !st.unread.is_empty() { return Err(ask_lines(&Status { missing: vec![], ..st })); }
    let wrote = match applied_doc(flow, &st) {
        Some(doc) => { std::fs::write(flow.join("parts.json"), serde_json::to_string_pretty(&doc).unwrap_or_default() + "\n").map_err(|e| vec![format!("parts.json: {e}")])?; true }
        None => false,
    };
    Ok(json!({"chips": st.chips, "sheets": st.sheets.keys().collect::<Vec<_>>(), "pins": st.sheets.values().map(|s| s.pins.len()).sum::<usize>(), "declined": st.declined.iter().map(|(r, w)| json!({"part": r, "reason": w})).collect::<Vec<_>>(), "missing": st.missing.iter().map(|c| c.r.clone()).collect::<Vec<_>>(), "partsJsonChanged": wrote}))
}

/// What `datasheet status` prints.
pub fn status_lines(flow: &Path) -> (String, Vec<String>) {
    let st = status(flow);
    let mut h: Vec<String> = st.sheets.values().map(|s| format!("{}: {} pins from {} ({})", s.part, s.pins.len(), s.cite, s.file)).collect();
    for (r, why) in &st.declined { h.push(format!("{r}: none available: {why}")); }
    h.extend(ask_lines(&st));
    if !applied(flow, &st) && st.bad.is_empty() { h.push("parts.json does not carry these pins yet: `datasheet apply` (or `auto`) writes them".into()); }
    (format!("{} identified chip(s): {} with datasheet pins, {} none available, {} waiting, {} file(s) refused", st.chips, st.sheets.len(), st.declined.len(), st.missing.len(), st.bad.len() + st.unread.len()), h)
}

/// Net names the datasheets give, by pin ("U2.6").
#[derive(Default, Clone)]
pub struct Names { by_pin: BTreeMap<String, Named> }

#[derive(Clone, Debug)]
pub struct Named { pub net: String, pub pin_name: String, pub ty: String, pub cite: String, rank: u8 }

/// Which pin names a conductor that several named pins sit on: ground first, then what drives it, then what listens.
fn rank(ty: &str) -> u8 { match ty { "ground" => 0, "power_out" => 1, "power_in" => 2, "output" => 3, "open_drain" => 4, "bidirectional" => 5, "input" => 6, _ => 7 } }

fn clean(name: &str) -> String {
    let s: String = name.chars().map(|c| if c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '_') { c } else { '_' }).collect();
    s.trim_matches('_').to_string()
}

/// The net a named pin gives its copper: the name the sheet states for the board (`net`), GND for a ground pin, else
/// `<ref>_<pin name>`, so no two chips' pins share a name unless a sheet says they do. A no-connect pin names nothing.
pub fn net_of(r: &str, p: &Pin) -> Option<String> {
    if p.ty == "no_connect" { return None; }
    if let Some(n) = &p.net { return Some(n.clone()); }
    if p.ty == "ground" { return Some("GND".into()); }
    let n = clean(&p.name);
    if n.is_empty() { None } else { Some(format!("{r}_{n}")) }
}

impl Names {
    pub fn from(st: &Status) -> Names {
        let mut by_pin = BTreeMap::new();
        for s in st.sheets.values() { for p in &s.pins { if let Some(net) = net_of(&s.part, p) { by_pin.insert(format!("{}.{}", s.part, p.pad), Named { net, pin_name: p.name.clone(), ty: p.ty.clone(), cite: s.cite.clone(), rank: rank(&p.ty) }); } } }
        Names { by_pin }
    }
    pub fn load(flow: &Path) -> Names { Names::from(&status(flow)) }
    pub fn get(&self, pin: &str) -> Option<&Named> { self.by_pin.get(pin) }
    pub fn is_empty(&self) -> bool { self.by_pin.is_empty() }
    /// The pin that names a conductor these pins sit on, and the name it gives.
    pub fn namer<'a>(&'a self, pins: impl IntoIterator<Item = &'a String>) -> Option<(&'a String, &'a Named)> {
        let mut best: Option<(&String, &Named)> = None;
        for q in pins { if let Some((k, n)) = self.by_pin.get_key_value(q) { if best.map(|b| (n.rank, sch_natural(k)) < (b.1.rank, sch_natural(b.0))).unwrap_or(true) { best = Some((k, n)); } } }
        best
    }
    pub fn name_for<'a>(&'a self, pins: impl IntoIterator<Item = &'a String>) -> Option<String> { self.namer(pins).map(|(_, n)| n.net.clone()) }
    /// Changes whenever a name the datasheets give changes: attribution runs again then.
    pub fn key(&self) -> String {
        if self.by_pin.is_empty() { return String::new(); }
        crate::confidence::sha256_hex(self.by_pin.iter().map(|(k, n)| format!("{k}={}", n.net)).collect::<Vec<_>>().join("\n").as_bytes())
    }
}

/// Order pins as people count them: U2.9 before U2.10.
fn sch_natural(pin: &str) -> (String, u64, String) {
    let (r, p) = pin.rsplit_once('.').unwrap_or((pin, ""));
    (r.to_string(), p.parse::<u64>().unwrap_or(u64::MAX), p.to_string())
}

/// After attribution: every net records the datasheet pins that sit on it, and a net that took its name from one is tagged
/// for what it is. The name is from the document (`nameSource` cites it, `nameTag` DATASHEET); the pins are on the net because
/// the copper joins them, which `copperEvidence` and `tag` MEASURED go on saying. Writes nets.json and net-evidence.json.
pub fn stamp(flow: &Path) -> Result<Value, String> {
    let names = Names::load(flow);
    let mut nets = items::read_json(&flow.join("nets.json"))?;
    let mut ev = items::read_json(&flow.join("net-evidence.json")).unwrap_or(Value::Null);
    let (mut named, mut carrying) = (vec![], 0);
    for n in nets["nets"].as_array_mut().into_iter().flatten() {
        let name = n["name"].as_str().unwrap_or("").to_string();
        let pins: Vec<String> = n["pins"].as_array().into_iter().flatten().filter_map(|p| p.as_str().map(str::to_string)).collect();
        let mut on: Vec<(&String, &Named)> = pins.iter().filter_map(|q| names.get(q).map(|d| (q, d))).collect();
        on.sort_by_key(|(q, d)| (d.rank, sch_natural(q)));
        let Some(o) = n.as_object_mut() else { continue };
        if on.is_empty() { o.remove("datasheetPins"); } else { carrying += 1; o.insert("datasheetPins".into(), json!(on.iter().map(|(q, d)| json!({"pin": q, "name": d.pin_name, "type": d.ty})).collect::<Vec<_>>())); }
        // only a net attribution made from copper takes a datasheet name; a net somebody stated keeps what they stated
        let by = on.iter().find(|(_, d)| d.net == name);
        let ours = o.get("fromCopper") == Some(&json!(true));
        match (ours, by) {
            (true, Some((q, d))) => {
                o.insert("nameSource".into(), json!(d.cite)); o.insert("source".into(), json!(d.cite)); o.insert("nameTag".into(), json!("DATASHEET")); o.insert("statedTag".into(), json!("DATASHEET"));
                o.insert("why".into(), json!(format!("the copper at {q} carries the name of that pin ({}, {}): the name is from the datasheet, the pins are joined as the copper shows", d.pin_name, d.ty.replace('_', " "))));
                // the name is documented; a join every pin of which is seen stays MEASURED, and a contradiction stays a finding
                if o.get("tag") == Some(&json!("INFERRED")) && !o.contains_key("contradiction") { o.insert("tag".into(), json!("DATASHEET")); }
                named.push(name.clone());
            }
            // a name an earlier sheet gave and no sheet gives now is not carried on as documented
            (true, None) if o.get("nameTag") == Some(&json!("DATASHEET")) => { for k in ["nameSource", "source", "nameTag", "statedTag"] { o.remove(k); } if o.get("tag") == Some(&json!("DATASHEET")) { o.insert("tag".into(), json!("INFERRED")); } o.insert("why".into(), json!("no name in the pin map: pins joined by copper")); }
            _ => {}
        }
        let (tag, src, dp) = (o.get("tag").cloned().unwrap_or(Value::Null), o.get("nameSource").cloned(), o.get("datasheetPins").cloned());
        if let Some(row) = ev["nets"].as_array_mut().into_iter().flatten().find(|r| r["net"].as_str() == Some(name.as_str())) {
            row["tag"] = tag;
            if named.last() == Some(&name) { if let Some(s) = src { row["nameSource"] = s; } row["nameTag"] = json!("DATASHEET"); }
            match dp { Some(d) => row["datasheetPins"] = d, None => { if let Some(r) = row.as_object_mut() { r.remove("datasheetPins"); } } }
        }
    }
    let w = |p: &Path, v: &Value| std::fs::write(p, serde_json::to_string_pretty(v).unwrap_or_default() + "\n").map_err(|e| format!("{}: {e}", p.display()));
    w(&flow.join("nets.json"), &nets)?;
    if ev.is_object() { w(&flow.join("net-evidence.json"), &ev)?; }
    Ok(json!({"netsNamedFromDatasheet": named, "netsCarryingDatasheetPins": carrying, "nets": nets["nets"].as_array().map(|a| a.len()).unwrap_or(0)}))
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::collections::BTreeSet;

    fn dir(name: &str) -> std::path::PathBuf {
        let d = std::env::temp_dir().join(format!("dspins-test-{name}-{}", std::process::id())).join("reverse-flow");
        let _ = std::fs::remove_dir_all(&d);
        std::fs::create_dir_all(d.join(DIR)).unwrap();
        d
    }
    fn put(d: &Path, f: &str, v: Value) { std::fs::write(d.join(f), v.to_string()).unwrap(); }
    #[test]
    fn a_position_with_no_part_fitted_is_not_asked_for_datasheet_pins() {
        let d = dir("notfitted");
        put(&d, "parts.json", json!({"parts": [
            {"ref": "U1", "value": "XY1234", "marking": "XY1234", "fieldTags": {"value": "SEEN"}},
            {"ref": "U2", "value": "not fitted", "marking": "", "fieldTags": {"value": "SEEN"}},
            {"ref": "U3", "value": "DNP", "fieldTags": {"value": "SEEN"}},
            {"ref": "Q1", "value": "AO3400", "mpn": "AO3400", "function": "pads only, unpopulated", "fieldTags": {"value": "SEEN"}},
            {"ref": "U4", "value": "LM000", "fitted": false, "fieldTags": {"value": "SEEN"}}]}));
        let st = status(&d);
        assert_eq!(st.missing.iter().map(|c| c.r.as_str()).collect::<Vec<_>>(), vec!["U1"], "only the chip that is there is asked about");
        assert_eq!(st.chips, 1);
        // words that only look alike do not count: a part number with NF or NC inside it is a part
        assert!(!not_fitted(&json!({"value": "NFC reader ST25", "mpn": "INF123NC7"})));
        assert!(not_fitted(&json!({"value": "0R (NF)"})) && not_fitted(&json!({"function": "Do Not Populate"})));
        std::fs::remove_dir_all(d.parent().unwrap()).ok();
    }

    fn sheet() -> Value {
        json!({"part": "U1", "mpn": "XY1234", "document": {"name": "XY1234 regulator", "number": "DS-1", "revision": "C", "page": 4},
            "pins": [{"pad": 1, "name": "VIN", "type": "power_in", "expects": [{"kind": "capacitor_to_ground"}]}, {"pad": "2", "name": "GND", "type": "ground"}, {"pad": "3", "name": "SW", "type": "output", "expects": [{"kind": "inductor"}, {"kind": "capacitor_to_pin", "pin": "BST"}]}, {"pad": "4", "name": "BST", "type": "passive"}, {"pad": "5", "name": "NC", "type": "no_connect"}]})
    }
    // parts with no footprint: the pads are not known, so only the file's own rules are checked
    fn parts() -> Value {
        json!({"parts": [
            {"ref": "U1", "value": "XY1234", "marking": "XY1234 831", "fieldTags": {"identity": "SEEN", "value": "SEEN"}},
            {"ref": "U2", "value": "regulator (marking unreadable)", "marking": "unreadable", "fieldTags": {"identity": "INFERRED", "value": "UNKNOWN"}},
            {"ref": "U3", "value": "ABC55", "mpn": "ABC55-Q1", "marking": "ABC55"},
            {"ref": "R1", "value": "10k", "marking": "103", "fieldTags": {"value": "SEEN"}}]})
    }

    #[test]
    fn a_sheet_is_read_cited_and_checked_against_the_footprint() {
        let pads: Vec<String> = (1..=5).map(|k| k.to_string()).collect();
        let s = parse(&sheet(), "U1", "datasheets/U1.pins.json", Some(&pads)).unwrap();
        assert_eq!(s.cite, "XY1234 regulator, DS-1, revision C, page 4");
        assert_eq!(s.pins.len(), 5);
        assert_eq!(s.pin("1").unwrap().name, "VIN");
        assert_eq!(s.pads_named("bst"), vec!["4".to_string()]);
        // a footprint with a pad the sheet does not name, and a sheet pad the footprint lacks: both are said
        let six: Vec<String> = ["1", "2", "3", "4", "6"].iter().map(|k| k.to_string()).collect();
        let e = parse(&sheet(), "U1", "f", Some(&six)).unwrap_err();
        assert!(e.iter().any(|l| l.contains("pad 5 (NC) is not on the footprint")) && e.iter().any(|l| l.contains("footprint pad(s) 6")));
        // no citation, an unknown type, an expectation naming no pin: refused, each with its reason
        let mut bad = sheet(); bad["document"] = json!({"name": "x"}); bad["pins"][0]["type"] = json!("supply"); bad["pins"][2]["expects"][1]["pin"] = json!("BOOT");
        let e = parse(&bad, "U1", "f", None).unwrap_err();
        assert!(e.iter().any(|l| l.contains("document.revision")) && e.iter().any(|l| l.contains("document.page")));
        assert!(e.iter().any(|l| l.contains("type \"supply\"")) && e.iter().any(|l| l.contains("names pin \"BOOT\"")));
    }

    #[test]
    fn only_chips_whose_marking_was_read_are_asked_for_and_a_reasoned_no_is_an_answer() {
        let d = dir("status");
        put(&d, "parts.json", parts());
        let st = status(&d);
        // U1 (read) and U3 (part number beside a marking) wait; U2's marking was not read; R1 is no chip
        assert_eq!(st.missing.iter().map(|c| c.r.as_str()).collect::<Vec<_>>(), vec!["U1", "U3"]);
        let lines = ask_lines(&st);
        assert!(lines[0].contains("datasheets/<ref>.pins.json") && lines[1].contains("\"available\": false"));
        assert!(lines.iter().any(|l| l.starts_with("U1: XY1234")) && !lines.iter().any(|l| l.starts_with("U2")));
        put(&d, "datasheets/U1.pins.json", sheet());
        // "none available" without a reason is no answer; with one it is
        put(&d, "datasheets/U3.pins.json", json!({"part": "U3", "available": false}));
        let st = status(&d);
        assert!(st.missing.is_empty() && st.sheets.contains_key("U1") && st.bad.len() == 1 && st.bad[0].1[0].contains("needs a \"reason\""));
        put(&d, "datasheets/U3.pins.json", json!({"part": "U3", "available": false, "reason": "ABC55 is a top mark shared by three parts"}));
        let st = status(&d);
        assert!(st.bad.is_empty() && st.declined == vec![("U3".to_string(), "ABC55 is a top mark shared by three parts".to_string())]);
        // pins for a chip nobody identified are refused: no witness from a guess
        put(&d, "datasheets/U2.pins.json", { let mut s = sheet(); s["part"] = json!("U2"); s });
        let st = status(&d);
        assert!(st.unread.len() == 1 && st.unread[0].contains("its marking was not read") && !st.sheets.contains_key("U2"));
        assert!(apply(&d).is_err());
        std::fs::remove_dir_all(d.parent().unwrap()).ok();
    }

    #[test]
    fn apply_writes_pin_names_and_kicad_types_once_and_takes_them_back_with_the_sheet() {
        let d = dir("apply");
        put(&d, "parts.json", parts());
        put(&d, "datasheets/U1.pins.json", sheet());
        put(&d, "datasheets/U3.pins.json", json!({"part": "U3", "available": false, "reason": "no public datasheet"}));
        assert!(!applied(&d, &status(&d)));
        let s = apply(&d).unwrap();
        assert_eq!(s["partsJsonChanged"], true);
        let p = items::read_json(&d.join("parts.json")).unwrap();
        assert_eq!(p["parts"][0]["pins"]["1"], json!({"name": "VIN", "type": "power_in"}));
        assert_eq!(p["parts"][0]["pins"]["2"], json!({"name": "GND", "type": "power_in"}));
        assert_eq!(p["parts"][0]["pins"]["5"], json!({"name": "NC", "type": "passive"}));
        assert_eq!(p["parts"][0]["pinsSource"], "XY1234 regulator, DS-1, revision C, page 4");
        assert!(p["parts"][1]["pins"].is_null() && p["parts"][2]["pins"].is_null());
        // two pads of one name that both drive are one output inside the chip: the second is passive, so the chip's own tie is no ERC conflict
        let mut two = sheet(); two["pins"][3] = json!({"pad": "4", "name": "SW", "type": "output"}); two["pins"][2]["expects"] = json!([]);
        put(&d, "datasheets/U1.pins.json", two);
        apply(&d).unwrap();
        let q = items::read_json(&d.join("parts.json")).unwrap();
        assert_eq!(q["parts"][0]["pins"]["3"]["type"], "output"); assert_eq!(q["parts"][0]["pins"]["4"], json!({"name": "SW", "type": "passive"}));
        put(&d, "datasheets/U1.pins.json", sheet());
        apply(&d).unwrap();
        // a second run changes nothing: the plan sees it as done
        assert!(applied(&d, &status(&d)));
        assert_eq!(apply(&d).unwrap()["partsJsonChanged"], false);
        // the sheet withdrawn: the pins it gave go with it
        put(&d, "datasheets/U1.pins.json", json!({"part": "U1", "available": false, "reason": "the marking fits two packages"}));
        apply(&d).unwrap();
        assert!(items::read_json(&d.join("parts.json")).unwrap()["parts"][0]["pins"].is_null());
        std::fs::remove_dir_all(d.parent().unwrap()).ok();
    }

    #[test]
    fn a_conductor_takes_its_name_from_the_pin_that_matters_most_and_the_join_stays_the_coppers() {
        let d = dir("names");
        put(&d, "parts.json", parts());
        put(&d, "datasheets/U1.pins.json", sheet());
        let names = Names::load(&d);
        let set = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::<BTreeSet<String>>();
        // an output and a passive pin on one conductor: the output names it; ground wins over everything; a part with no sheet names nothing
        assert_eq!(names.name_for(&set(&["U1.4", "U1.3", "C9.1"])).as_deref(), Some("U1_SW"));
        assert_eq!(names.name_for(&set(&["U1.1", "U1.2"])).as_deref(), Some("GND"));
        assert_eq!(names.name_for(&set(&["U1.5"])), None);
        assert_eq!(names.name_for(&set(&["C9.1", "R1.2"])), None);
        assert!(!names.key().is_empty());
        // after attribution: the net made from copper is tagged by its name's source, a net somebody stated is left as stated
        put(&d, "nets.json", json!({"nets": [
            {"name": "U1_SW", "fromCopper": true, "tag": "MEASURED", "pins": ["L1.1", "U1.3"], "why": "no name in the pin map: pins joined by copper", "tests": ["T-pour-nets"]},
            {"name": "U1_VIN", "fromCopper": true, "tag": "INFERRED", "pins": ["U1.1"], "why": "no name in the pin map: pins joined by copper", "tests": ["T-pour-nets"]},
            {"name": "VOUT", "tag": "INFERRED", "pins": ["U1.4", "C3.1"], "why": "typed", "tests": ["T-net-VOUT"]},
            {"name": "CU_R1_1", "fromCopper": true, "tag": "MEASURED", "pins": ["R1.1", "R2.1"]}]}));
        put(&d, "net-evidence.json", json!({"nets": [{"net": "U1_SW", "tag": "MEASURED", "nameSource": "copper"}, {"net": "U1_VIN", "tag": "INFERRED", "nameSource": "copper"}]}));
        let s = stamp(&d).unwrap();
        assert_eq!(s["netsNamedFromDatasheet"], json!(["U1_SW", "U1_VIN"]));
        let n = items::read_json(&d.join("nets.json")).unwrap();
        assert_eq!(n["nets"][0]["tag"], "MEASURED"); assert_eq!(n["nets"][0]["nameTag"], "DATASHEET");
        assert_eq!(n["nets"][0]["nameSource"], "XY1234 regulator, DS-1, revision C, page 4");
        assert_eq!(n["nets"][0]["datasheetPins"], json!([{"pin": "U1.3", "name": "SW", "type": "output"}]));
        assert_eq!(n["nets"][1]["tag"], "DATASHEET"); assert_eq!(n["nets"][1]["source"], "XY1234 regulator, DS-1, revision C, page 4");
        assert_eq!(n["nets"][2]["tag"], "INFERRED"); assert_eq!(n["nets"][2]["why"], "typed"); assert!(n["nets"][2]["nameTag"].is_null());
        assert_eq!(n["nets"][2]["datasheetPins"][0]["name"], "BST");
        assert!(n["nets"][3]["datasheetPins"].is_null());
        let e = items::read_json(&d.join("net-evidence.json")).unwrap();
        assert_eq!(e["nets"][0]["nameTag"], "DATASHEET"); assert_eq!(e["nets"][1]["tag"], "DATASHEET");
        // the sheet withdrawn and attribution run again under the old name: the net is no longer called documented
        std::fs::remove_file(d.join("datasheets/U1.pins.json")).unwrap();
        stamp(&d).unwrap();
        let n = items::read_json(&d.join("nets.json")).unwrap();
        assert!(n["nets"][1]["nameTag"].is_null() && n["nets"][1]["tag"] == "INFERRED" && n["nets"][0]["tag"] == "MEASURED");
        std::fs::remove_dir_all(d.parent().unwrap()).ok();
    }
}