AI Flow

#!/usr/bin/env python3
"""The board's project page, revision 1, before any EDA work: the parts the run chose (each one a
wiki component page), a README that says what the board is and lists them, and a KiCad Plugin and
Content Manager (PCM) repository on the page so KiCad installs the whole parts library in one go.

  python3 -c "$(cat project-page.py)" <config.json>

config: {"run": dir, "page": "owner/slug", "title", "brief", "summary", "parts": [{refs, mpn,
manufacturer, value, wiki}], "create": bool, "push": bool, "localOnly": bool (no wiki call at all: project-library parts only), "private": bool, "version": "1.0.0" (the first build's; a later build whose package content changed gets the next patch version, issue #169),
"pageDir": "rig/boards/<board>" or null, "molecule": bool}

molecule (issues #137, #158): the spec declares a molecule (or the requirements say formFactor.kind "molecule"), so a page this makes is of kind molecule, not board.
Its model, footprint and symbol are built by the wiki's model registry from the sources molecule-publish
puts there after export; nothing here pushes or binds a GLB.

pageDir (issue #85): the board lives in a folder of an EXISTING page (one page per project). Its README
and PCM repository go to <pageDir>/README.md and <pageDir>/pcm/, the repository URLs carry the folder,
and the page is never created (an existing page is implied). The library nickname, the PCM identifier and
the Cloud Library name carry the folder too, so two boards on one page never share a library. The
folder's README is pushed when the folder has none yet, or with replaceReadme.

The parts library comes from Adom Cloud Libraries (`adom-wiki library`): one library per board,
the parts added by their wiki pages, exported as a KiCad bundle, then repackaged in the PCM layout
(metadata.json, symbols/<Lib>.kicad_sym, footprints/<Lib>.pretty, 3dmodels/<Lib>.3dshapes) with
the footprints' 3D paths pointed at ${KICAD10_3RD_PARTY}/3dmodels/<pkgid_>/<Lib>.3dshapes/, which is
where PCM puts the files (adom/kicad-pcm documents the layout). Prints one JSON object.

The library is made once per board: its lib_ id is kept in the run (<run>/project-page/cloud-library.json and
the config's libRef) and reused, so a second run never makes a second library of the same name. Adds that
the wiki rate-limits (HTTP 429, retry_after) are retried with backoff; the export must then hold every part
(each distinct part's symbol, the footprint it names and that footprint's model resolve; parts may share a
footprint or a model), or the step is an ERROR. Parts
from the board's own project library (row "projectLib", `parts --project-lib`) are packaged beside them.
A wiki part whose footprint names no 3D model takes its own page's STEP, bound in
the footprint, only when the page names exactly one STEP for it and the file checks out (bind_page_models);
otherwise the part stays a gap and the error says why.
The symbol file gets one current header (SYM_VERSION) and is parse-checked (structurally, and with
`kicad-cli sym upgrade` when ADOM_AIFLOW_KICAD_CLI or kicad-cli is there) before anything is published."""
import hashlib, json, os, re, shutil, subprocess, sys, tempfile, time, zipfile

API = "https://wiki.adom.inc/api/pages"
# KiCad 9's symbol library format (KiCad 10 reads it; KiCad 10 itself writes 20251024). The Cloud Library
# export wrote 20211014 over symbols in newer syntax, which KiCad 10 refuses as a whole library (issue #64).
SYM_VERSION = 20241209
KICAD10_SYM_VERSION = 20251024
ADD_CHUNK = 10



def mpn_candidates(p):
    """The MPN, then the MPN without a packaging suffix (MCP73831T-2ACI/OT -> MCP73831T-2ACI) when the
    parts record matched the page that way (wikiMatch), so the page's symbol is found by its own name."""
    out = [p["mpn"]]
    if p.get("wiki") and "/" in p["mpn"]:
        out.append(p["mpn"].split("/")[0])
    return out

def clip(text, n=600):
    """Error text within n characters, keeping its beginning (the cause) and its end (issue #76)."""
    text = text or ""
    if len(text) <= n:
        return text
    head = int(n * 0.7)
    return text[:head] + " ... " + text[len(text) - (n - head - 5):]


def fail(msg, **extra):
    raise SystemExit(json.dumps(dict({"ok": False, "error": msg}, **extra)))


def sh(args, check=True):
    p = subprocess.run(args, capture_output=True, text=True, stdin=subprocess.DEVNULL)
    if check and p.returncode != 0:
        raise SystemExit(json.dumps({"ok": False, "error": f"{' '.join(args[:4])}: {clip((p.stderr or p.stdout).strip(), 600)}"}))
    return p.stdout


def wjson(args):
    out = sh(args + ["--json"])
    try:
        return json.loads(out)
    except Exception:
        return {}


def sha(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for b in iter(lambda: f.read(1 << 20), b""): h.update(b)
    return h.hexdigest()


def check_dir(d):
    """A page folder: relative, no '..' or '.' parts, no leading slash, no backslash (the binary checks it too)."""
    d = (d or "").strip().rstrip("/")
    if not d or d == ".":
        return None
    parts = d.split("/")
    if d.startswith("/") or "\\" in d or ":" in d or any(p in ("", ".", "..") for p in parts):
        fail(f"pageDir {d!r} must be a folder of the page, relative to its root (like rig/boards/rig-carrier)")
    return d


def under(d, rel):
    return f"{d}/{rel}" if d else rel


def push_files(cfg, work, folder, files, message):
    """adom-wiki repo push of files made in `work`, staged under the page folder; returns the commit."""
    cwd = work
    if folder:
        cwd = os.path.join(work, "push-stage"); shutil.rmtree(cwd, ignore_errors=True)
        for f in files:
            dst = os.path.join(cwd, folder, f); os.makedirs(os.path.dirname(dst), exist_ok=True); shutil.copy(os.path.join(work, f), dst)
    p = subprocess.run(["adom-wiki", "repo", "push", cfg["page"], "--files", *[under(folder, f) for f in files], "-m", message],
                       capture_output=True, text=True, cwd=cwd, stdin=subprocess.DEVNULL)
    m = re.search(r'"commit":\s*"([0-9a-f]{7,})"', p.stdout)
    if p.returncode != 0 or not m:
        raise SystemExit(json.dumps({"ok": False, "error": "push failed: " + clip(p.stderr or p.stdout, 600)}))
    return m.group(1)


def readme_wanted(cfg, folder, created):
    """README rev 1 goes on a page this step made (or when asked); an existing page's README is never
    replaced unasked. In a page folder: when the folder has no README yet."""
    if created or cfg.get("replaceReadme"):
        return True
    if folder:
        return subprocess.run(["adom-wiki", "repo", "show", cfg["page"], under(folder, "README.md")], capture_output=True, stdin=subprocess.DEVNULL).returncode != 0
    return False


def lib_name(slug):
    return re.sub(r"[^A-Za-z0-9_-]", "_", slug)


def pkg_id(owner, slug):
    return "inc.adom.wiki." + re.sub(r"[^a-z0-9.-]", "-", f"{owner}.{slug}".lower())


def payload(v, key):
    """adom-wiki wraps replies in {status, data} envelopes, sometimes twice: the first `key` found going down."""
    while isinstance(v, dict):
        if key in v:
            return v[key]
        v = v.get("data")
    return None


def retry_after(text):
    """Seconds to wait when adom-wiki says HTTP 429 / RATE_LIMITED, else None."""
    if not re.search(r"\b429\b|RATE_LIMITED|rate.?limit", text, re.I):
        return None
    m = re.search(r"retry[_ -]?after\W{0,4}(\d+(?:\.\d+)?)", text, re.I)
    return float(m.group(1)) if m else -1.0


def run_retrying(args, attempts=None, log=None):
    """Run an adom-wiki call, honouring 429 retry_after with bounded retries and exponential backoff.
    Returns (ok, output, waits)."""
    attempts = attempts or int(os.environ.get("ADOM_AIFLOW_WIKI_ATTEMPTS", "8"))
    cap = float(os.environ.get("ADOM_AIFLOW_WIKI_MAX_WAIT", "300"))
    base = float(os.environ.get("ADOM_AIFLOW_WIKI_BACKOFF", "5"))
    waits = []
    out = ""
    for i in range(attempts):
        p = subprocess.run(args, capture_output=True, text=True, stdin=subprocess.DEVNULL)
        out = (p.stdout or "") + (p.stderr or "")
        ra = retry_after(out)
        if p.returncode == 0 and ra is None:
            return True, out, waits
        if ra is None or i == attempts - 1:
            return False, out, waits
        # the server's retry_after, never less than the backoff (base, 2 base, 4 base ...), at most cap
        wait = min(cap, max(ra if ra >= 0 else 0.0, base * (2 ** i)))
        waits.append(wait)
        time.sleep(wait)
    return False, out, waits


def page_exists(ref):
    return subprocess.run(["adom-wiki", "repo", "show", ref, "page.json"], capture_output=True, stdin=subprocess.DEVNULL).returncode == 0


def libraries():
    return payload(wjson(["adom-wiki", "library", "list"]), "libraries") or []


def lib_ref_of(lib):
    return (lib or {}).get("ref") or (lib or {}).get("id")


def board_library(name, recorded, work):
    """The board's one Cloud Library: the lib_ id the run recorded, else the one library of that name,
    else a new one (recorded at once, so a failed run never leads to a second library)."""
    rec_path = os.path.join(work, "cloud-library.json")
    if not recorded and os.path.isfile(rec_path):
        try:
            recorded = json.load(open(rec_path)).get("ref")
        except Exception:
            recorded = None
    items = libraries()
    notes = []
    lib = next((l for l in items if recorded and lib_ref_of(l) == recorded), None)
    if lib is None:
        same = [l for l in items if l.get("name") == name]
        if recorded:
            notes.append(f"the recorded library {recorded} is gone; " + ("using the one named " + name if same else "making a new one"))
        if len(same) > 1:
            # duplicates from an earlier release: keep the fullest (then the oldest), never add another
            same.sort(key=lambda l: (-(l.get("item_count") or 0), l.get("created_at") or ""))
            notes.append(f"{len(same)} libraries are named {name}: using {lib_ref_of(same[0])} (the fullest); delete the others with `adom-wiki library delete <lib_id> --confirm`")
        lib = same[0] if same else None
    if lib is None:
        # creating a library makes it the user's active one: put their own active library back after
        was_active = next((lib_ref_of(l) for l in items if l.get("active")), None)
        created = wjson(["adom-wiki", "library", "create", name])
        ref = payload(created, "ref") or payload(created, "id")
        if was_active:
            sh(["adom-wiki", "library", "activate", was_active], check=False)
        if not ref:
            lib = next((l for l in libraries() if l.get("name") == name), None)
            ref = lib_ref_of(lib)
        if not ref:
            fail(f"adom-wiki library create {name} gave no lib_ id")
        lib = {"ref": ref, "name": name}
    ref = lib_ref_of(lib)
    json.dump({"ref": ref, "name": name}, open(rec_path, "w"))
    return ref, notes


def drop_stale_items(ref, pages):
    """A reused library holds this BOM and nothing else (issue #125): a part taken off the BOM must not stay
    in the export. Runs before the adds, so an item a renamed page no longer matches is simply added back.
    Returns (removed labels, retry waits); only this board's own library ever reaches here."""
    ok, out, waits = run_retrying(["adom-wiki", "library", "show", ref, "--json"])
    try:
        items = payload(json.loads(out[out.index("{"):out.rindex("}") + 1]), "items") if ok else None
    except ValueError:
        items = None
    if not isinstance(items, list):
        fail(f"could not read the Cloud Library {ref} to compare it with the BOM: " + clip(out.strip(), 300), cloudLibrary=ref)
    want = {p.lower() for p in pages}
    removed, failed = [], []
    for it in items:
        page = f"{it.get('owner')}/{it.get('slug')}" if it.get("owner") and it.get("slug") else None
        if (page and page.lower() in want) or str(it.get("ref", "")).lower() in want:
            continue
        ok, out, w = run_retrying(["adom-wiki", "library", "remove", ref, str(it.get("id")), "--json"])
        waits += w
        label = f"{it.get('mpn') or it.get('title') or it.get('ref')} ({page or 'page gone'})"
        (removed if ok else failed).append(label)
    if failed:
        fail(f"parts no longer on the BOM could not be removed from the Cloud Library {ref}: " + ", ".join(failed), cloudLibrary=ref)
    return removed, waits


def cloud_library(name, parts, work, recorded=None):
    """Reuse (or create once) the board's Cloud Library, make it hold this BOM (dropping parts no longer on
    it, adding every part page with 429 retries), export the KiCad bundle."""
    ref, notes = board_library(name, recorded, work)
    pages = list(dict.fromkeys(p["wiki"] for p in parts if p.get("wiki")))
    removed, waits = drop_stale_items(ref, pages)
    if removed:
        notes.append("removed from the Cloud Library, no longer on the BOM: " + ", ".join(removed))
    failed = []
    for i in range(0, len(pages), ADD_CHUNK):
        chunk = pages[i:i + ADD_CHUNK]
        ok, out, w = run_retrying(["adom-wiki", "library", "add", ref, *chunk, "--json"])
        waits += w
        if not ok:
            # one page at a time, so the report names the pages that really failed
            for pg in chunk:
                ok1, out1, w1 = run_retrying(["adom-wiki", "library", "add", ref, pg, "--json"])
                waits += w1
                if not ok1:
                    failed.append({"page": pg, "error": clip(out1.strip(), 300)})
    if failed:
        fail(f"{len(failed)} of {len(pages)} part pages could not be added to the Cloud Library {ref} " +
             f"(after retries{' honouring retry_after' if waits else ''}): " + ", ".join(f["page"] for f in failed),
             cloudLibrary=ref, failedAdds=failed, notes=notes)
    bundle = os.path.join(work, "cloud-library.zip")
    if os.path.exists(bundle):
        os.remove(bundle)
    ok, out, w = run_retrying(["adom-wiki", "library", "export", ref, "-o", bundle, "--json"])
    waits += w
    try:
        report = json.loads(out[out.index("{"):]) if "{" in out else {}
    except Exception:
        report = {}
    if not ok or not os.path.isfile(bundle):
        fail("library export wrote no bundle: " + clip(out.strip(), 400), cloudLibrary=ref)
    return ref, bundle, report, {"addedPages": len(pages), "removedStale": removed, "rateLimitWaits": waits, "notes": notes}


# --- S-expressions: enough to count, extract and parse-check KiCad library files ---

def sx_tokens(text):
    i, n = 0, len(text)
    while i < n:
        c = text[i]
        if c in " \t\r\n":
            i += 1
        elif c in "()":
            yield c, i, i + 1; i += 1
        elif c == '"':
            j = i + 1
            while j < n and text[j] != '"':
                j += 2 if text[j] == "\\" else 1
            if j >= n:
                raise ValueError(f"unterminated string at offset {i}")
            yield "s", i, j + 1; i = j + 1
        else:
            j = i
            while j < n and text[j] not in ' \t\r\n()"':
                j += 1
            yield "a", i, j; i = j


def sx_parse(text):
    """Nested lists [tag, ...] of atoms (strings unquoted); each list also records (start, end) in .span."""
    class L(list):
        span = (0, 0)
    stack, root = [], None
    for kind, a, b in sx_tokens(text):
        if kind == "(":
            l = L(); l.span = (a, None); stack.append(l)
        elif kind == ")":
            if not stack:
                raise ValueError(f"unbalanced ')' at offset {a}")
            l = stack.pop(); l.span = (l.span[0], b)
            if stack:
                stack[-1].append(l)
            elif root is None:
                root = l
            else:
                raise ValueError("more than one top-level expression")
        else:
            v = text[a + 1:b - 1].replace('\\"', '"') if kind == "s" else text[a:b]
            if not stack:
                raise ValueError(f"atom outside any list at offset {a}")
            stack[-1].append(v)
    if stack:
        raise ValueError(f"{len(stack)} unclosed '(' (truncated file?)")
    if root is None:
        raise ValueError("empty file")
    return root


def sym_header_version(text):
    m = re.search(r"\(kicad_symbol_lib\s*\(version\s+(\d+)\)", text)
    return int(m.group(1)) if m else None


def normalise_sym_header(text, version):
    """One current header: the export's (version 20211014) under KiCad 9 syntax is refused by KiCad 10."""
    return re.sub(r"(\(kicad_symbol_lib\s*\(version\s+)(\d+)(\))", lambda m: f"{m.group(1)}{version}{m.group(3)}", text, count=1)


def unit_names_follow_parent(text):
    """KiCad names a symbol's units <parent>_<unit>_<style> and refuses a library whose units carry another name
    (a wiki symbol 'Lib:C_1nF_0402' renamed to its MPN keeps units 'C_1nF_0402_0_1'). Rename every top-level
    symbol's units to follow the symbol."""
    root = sx_parse(text)
    edits = []
    for l in root[1:]:
        if not (isinstance(l, list) and l[:1] == ["symbol"] and len(l) > 1 and isinstance(l[1], str)):
            continue
        parent = l[1]
        s, e = l.span
        for m in re.finditer(r'\(symbol\s+"([^"]+)_(\d+)_(\d+)"', text[s:e]):
            if m.start() == 0 or m.group(1) == parent:
                continue
            edits.append((s + m.start(1), s + m.end(1), parent))
    for a, b, parent in sorted(edits, reverse=True):
        text = text[:a] + parent + text[b:]
    return text


_ATOM = r'[^\s()"]+'
_QSTR = r'"(?:[^"\\]|\\.)*"'


def quote_atoms(text):
    """KiCad reads a symbol's name, its property names and values, and (extends parent) with or without quotes
    (issue #196: wiki symbols written `(property Footprint lib:name ...)`). The rest of this script reads and
    rewrites the quoted form KiCad itself writes, so put the quotes in once, where a library is read."""
    text = re.sub(r'\((symbol|extends)\s+(' + _ATOM + r')(?=[\s)])', lambda m: f'({m.group(1)} "{m.group(2)}"', text)
    text = re.sub(r'\(property\s+(' + _ATOM + r')(?=\s)', lambda m: f'(property "{m.group(1)}"', text)
    return re.sub(r'\(property\s+(' + _QSTR + r')\s+(' + _ATOM + r')(?=[\s)])', lambda m: f'(property {m.group(1)} "{m.group(2)}"', text)


def top_symbols(text):
    """{name: (start, end)} of the library's top-level symbols."""
    root = sx_parse(text)
    return {l[1]: l.span for l in root[1:] if isinstance(l, list) and l and l[0] == "symbol" and len(l) > 1}


def structural_check(sym_path, fp_dir):
    """The packaged library parses: one kicad_symbol_lib with a version, named symbols (no duplicates), and
    every footprint one footprint expression. Returns a list of problems."""
    problems = []
    try:
        t = open(sym_path).read()
        root = sx_parse(t)
        if root[:1] != ["kicad_symbol_lib"]:
            problems.append("the symbol file is not a kicad_symbol_lib")
        ver = next((l[1] for l in root[1:] if isinstance(l, list) and l[:1] == ["version"] and len(l) > 1), None)
        if not (ver and ver.isdigit()):
            problems.append("the symbol library has no version")
        names = []
        for l in root[1:]:
            if not isinstance(l, list):
                problems.append(f"stray atom {l!r} at the top of the symbol library")
            elif l[:1] == ["symbol"]:
                if len(l) < 2 or isinstance(l[1], list):
                    problems.append("a symbol without a name")
                else:
                    names.append(l[1])
            elif l[:1] not in (["version"], ["generator"], ["generator_version"]):
                problems.append(f"unexpected top-level ({l[0] if l else ''}) in the symbol library")
        dups = sorted({n for n in names if names.count(n) > 1})
        if dups:
            problems.append("duplicate symbols: " + ", ".join(dups))
    except Exception as e:
        problems.append(f"symbol library does not parse: {e}")
    for f in sorted(os.listdir(fp_dir)) if os.path.isdir(fp_dir) else []:
        try:
            r = sx_parse(open(os.path.join(fp_dir, f)).read())
            if r[:1] not in (["footprint"], ["module"]):
                problems.append(f"{f} is not a footprint")
        except Exception as e:
            problems.append(f"{f} does not parse: {e}")
    return problems


def kicad_check(sym_path):
    """`kicad-cli sym upgrade` loads the whole library the way KiCad does. (status, detail): status is
    "pass", "fail" or "unavailable" (no kicad-cli, or it could not run: the structural parse stands)."""
    kc = os.environ.get("ADOM_AIFLOW_KICAD_CLI") or shutil.which("kicad-cli")
    if not kc:
        return "unavailable", "no kicad-cli (set ADOM_AIFLOW_KICAD_CLI)"
    out = os.path.join(tempfile.mkdtemp(prefix="aiflow-symcheck-"), "check.kicad_sym")
    try:
        p = subprocess.run([kc, "sym", "upgrade", "--force", "-o", out, sym_path], capture_output=True, text=True, stdin=subprocess.DEVNULL,
                           timeout=int(os.environ.get("ADOM_AIFLOW_KICAD_CLI_TIMEOUT", "180")))
    except (OSError, subprocess.TimeoutExpired) as e:
        return "unavailable", f"{kc}: {e}"
    text = ((p.stdout or "") + (p.stderr or "")).strip()
    if p.returncode == 0 and os.path.isfile(out) and not re.search(r"unable to load|failed to load|error", text, re.I):
        return "pass", clip(text, 200)
    if re.search(r"load|pars|library|symbol", text, re.I):
        return "fail", clip(text, 400)
    return "unavailable", f"{kc} exit {p.returncode}: {clip(text, 200)}"


def broken_symbols(sym_path, parts=()):
    """Which symbols stop KiCad loading the library (issue #125): each one is loaded alone, with its parent
    when it extends one. Returns [{symbol, mpn, page, label}]; empty when each loads alone, or when no
    kicad-cli can say."""
    text = open(sym_path).read()
    syms = top_symbols(text)
    if not syms:
        return []
    header = text[:min(a for a, _ in syms.values())]
    by_norm = {}
    for p in parts:
        for key in (p.get("mpn"), (p.get("wiki") or "").rsplit("/", 1)[-1]):
            if key:
                by_norm.setdefault(norm(key), p)
    work = tempfile.mkdtemp(prefix="aiflow-symblame-")
    bad = []
    try:
        for name, (a, b) in syms.items():
            blocks, cur, seen = [], name, set()
            while cur in syms and cur not in seen:
                seen.add(cur)
                blocks.insert(0, text[syms[cur][0]:syms[cur][1]])
                m = re.search(r'\(extends\s+"?([^")\s]+)"?', text[syms[cur][0]:syms[cur][1]])
                cur = m.group(1) if m else None
            one = os.path.join(work, "one.kicad_sym")
            open(one, "w").write(header + "\n".join(blocks) + "\n)\n")
            status, _ = kicad_check(one)
            if status == "fail":
                p = by_norm.get(norm(name)) or {}
                label = f"{p.get('mpn') or name}" + (f" ({p['wiki']})" if p.get("wiki") else "")
                bad.append({"symbol": name, "mpn": p.get("mpn"), "page": p.get("wiki"), "label": label})
    finally:
        shutil.rmtree(work, ignore_errors=True)
    return bad


def norm(s):
    return re.sub(r"[^A-Z0-9]", "", (s or "").upper())


def merge_project_parts(parts, sym_text, fp_dir, m_dir, lib, pkgid):
    """Add the project-library parts (issue #66): each symbol (with its parent when it extends one), its
    footprint with the model path pointed where PCM installs it, and its STEP model. Returns the new text."""
    blocks, have = [], set(top_symbols(sym_text)) if sym_text else set()
    for p in parts:
        pl = p["projectLib"]
        src = quote_atoms(open(pl["symbolFile"]).read())
        spans = top_symbols(src)
        want = [pl["symbol"]]
        root = sx_parse(src)
        for l in root[1:]:
            if isinstance(l, list) and l[:2] == ["symbol", pl["symbol"]]:
                ext = next((x[1] for x in l if isinstance(x, list) and x[:1] == ["extends"] and len(x) > 1), None)
                if ext:
                    want.insert(0, ext)
        for name in want:
            if name in have:
                if name == pl["symbol"]:
                    fail(f"project library symbol {name} ({p['mpn']}) has the same name as a symbol already in the library")
                continue
            if name not in spans:
                fail(f"part {p['mpn']} is in the parts record but its symbol {name} is missing from {pl['symbolFile']}: run `parts` again")
            a, b = spans[name]
            block = src[a:b]
            if name == pl["symbol"]:
                block = re.sub(r'(\(property\s+"Footprint"\s+")[^"]*(")', lambda m: f'{m.group(1)}{lib}:{pl["footprint"]}{m.group(2)}', block, count=1)
                if '"Footprint"' not in block:
                    block = block.replace(f'(symbol "{name}"', f'(symbol "{name}" (property "Footprint" "{lib}:{pl["footprint"]}" (at 0 0 0) (effects (hide yes)))', 1)
            blocks.append(block); have.add(name)
        model = os.path.basename(pl["model"])
        shutil.copy(pl["model"], os.path.join(m_dir, model))
        t = open(pl["footprintFile"]).read()
        target = f'${{KICAD10_3RD_PARTY}}/3dmodels/{pkgid.replace(".", "_")}/{lib}.3dshapes/{model}'
        if re.search(r"\(model\s", t):
            t = re.sub(r'(\(model\s+)("[^"]*"|[^\s()]+)', lambda m: f'{m.group(1)}"{target}"', t, count=1)
        else:
            t = t.rstrip().rstrip(")") + f'\n  (model "{target}")\n)\n'
        open(os.path.join(fp_dir, pl["footprint"] + ".kicad_mod"), "w").write(t)
    if not blocks:
        return sym_text
    if not sym_text:
        sym_text = f'(kicad_symbol_lib (version {SYM_VERSION}) (generator "adom_aiflow")\n)\n'
    i = sym_text.rstrip().rfind(")")
    return sym_text[:i].rstrip() + "\n  " + "\n  ".join(blocks) + "\n)\n"


def sym_property(sym_text, syms, name, prop, seen=()):
    """A symbol's property value, following (extends "parent") when the symbol itself has none."""
    a, b = syms[name]
    block = sym_text[a:b]
    m = re.search(r'\(property\s+"?' + re.escape(prop) + r'"?\s+(?:"([^"]*)"|([^\s()"]+))', block)
    if m:
        return m.group(1) if m.group(1) is not None else m.group(2)
    e = re.search(r'\(extends\s+"?([^"\s()]+)"?', block)
    if e and e.group(1) in syms and e.group(1) not in seen:
        return sym_property(sym_text, syms, e.group(1), prop, seen + (name,))
    return ""


def distinct_parts(cfg_parts):
    """One entry per distinct part (its wiki page, else its project-library MPN), references merged."""
    first = {}
    for p in cfg_parts:
        key = p.get("wiki") or (p["mpn"] if p.get("projectLib") else None)
        if not key:
            continue
        e = first.setdefault(key, dict(p, refs=[]))
        e["refs"] = e["refs"] + list(p.get("refs") or [])
        if p.get("noModelExpected"):
            e["noModelExpected"] = True
    return first


def footprint_models(text):
    """The model files a footprint names (base names) and whether it is board_only."""
    refs = [m.group(1) or m.group(2) for m in re.finditer(r'\(model\s+(?:"([^"]*)"|([^\s()]+))', text)]
    return [os.path.basename(r.replace("\\", "/")) for r in refs], bool(re.search(r"\(attr\b[^)]*\bboard_only\b", text))


def report_names(report):
    """The export report's own symbol names for each MPN (the symbol may be named for the page, not the MPN)."""
    rep = payload(report, "report") or report or {}
    names = {}
    for x in payload(rep, "parts") or []:
        if isinstance(x, dict) and x.get("mpn") and x.get("name"):
            names.setdefault(norm(x["mpn"]), []).append(x["name"])
    return names


def symbol_of(p, syms, by_norm, rep_names):
    """The part's symbol in the packaged library, or None."""
    if p.get("projectLib") and not p.get("wiki"):
        return p["projectLib"]["symbol"] if p["projectLib"]["symbol"] in syms else None
    return next((by_norm[norm(c)] for c in mpn_candidates(p) + rep_names.get(norm(p["mpn"]), []) if norm(c) in by_norm), None)


# --- Route (b) for a footprint with no 3D model ---

def page_step_name(page_json):
    """The one STEP a wiki component page names for its footprint: page.json component.parts.step and
    model_3d_step, and any models_3d file of role "eda". The plain (unetched) variant the page names beside
    it (parts.model_3d_step_plain, models_3d role "plain") is a different file and never counts.
    Returns (file, the sha256 the page records for it or None, None), or (None, None, why)."""
    c = page_json.get("component") if isinstance(page_json, dict) else None
    c = c if isinstance(c, dict) else {}
    parts = c.get("parts") if isinstance(c.get("parts"), dict) else {}
    recs = [m for m in (c.get("models_3d") or []) if isinstance(m, dict) and isinstance(m.get("file"), str)]
    plain = {parts.get("model_3d_step_plain")} | {m["file"] for m in recs if m.get("role") == "plain"}
    named = [parts.get("step"), parts.get("model_3d_step")] + [m["file"] for m in recs if m.get("role") == "eda"]
    named = list(dict.fromkeys(f for f in named if isinstance(f, str) and f.lower().endswith((".step", ".stp")) and f not in plain))
    if not named:
        return None, None, "its page names no STEP model for the footprint (page.json component.parts.step)"
    if len(named) > 1:
        return None, None, f"its page names {len(named)} STEP models for the footprint ({', '.join(named)}): which one is this part's is not certain"
    f = named[0]
    rec = next((m["sha256"].lower() for m in recs if m["file"] == f and isinstance(m.get("sha256"), str) and m["sha256"]), None)
    return f, rec, None


def read_page_json(page):
    """A wiki page's page.json, or (None, why)."""
    ok, out, _ = run_retrying(["adom-wiki", "repo", "show", page, "page.json"])
    if not ok:
        return None, clip(out.strip(), 200)
    try:
        return json.loads(out[out.index("{"):out.rindex("}") + 1]), None
    except ValueError:
        return None, "page.json does not parse"


def fetch_page_file(page, path, dest):
    """One file of a wiki page into dest. (ok, detail)."""
    ok, out, _ = run_retrying(["adom-wiki", "repo", "show", page, path, "--out", dest])
    return ok and os.path.isfile(dest) and os.path.getsize(dest) > 0, clip(out.strip(), 200)


def is_step_file(path):
    with open(path, "rb") as f:
        return f.read(64).lstrip(b"\xef\xbb\xbf \t\r\n").startswith(b"ISO-10303-21")


def add_model_line(text, target):
    """The footprint with (model "<target>") added before its own closing parenthesis (it names none):
    no offset, scale or rotation, the identity binding the Adom component pages record for their STEP."""
    end = sx_parse(text).span[1] - 1
    return text[:end].rstrip() + f'\n  (model "{target}")\n' + text[end:]


def bind_page_models(cfg_parts, report, sym_path, fp_dir, m_dir, lib, pkgid):
    """A wiki part whose footprint names no 3D model gets its own wiki page's STEP, packaged in the library and
    bound in the footprint, only when that is certain: the page names exactly one STEP for the footprint
    (page_step_name), the file is on the page and is a STEP, it is the file the page records (when the page
    records a sha256), and every part sharing that footprint names the same model. Anything less binds
    nothing and says why: completeness then reports the part as before, with the reason. The bound model is
    checked like any other (`models` on the board: resolution, orientation, seating).
    Returns {"bound": [...], "notBound": {part key: why}}."""
    out = {"bound": [], "notBound": {}}
    sym_text = open(sym_path).read() if os.path.isfile(sym_path) else ""
    if not sym_text or not os.path.isdir(fp_dir):
        return out
    syms = top_symbols(sym_text)
    by_norm = {}
    for n in syms:
        by_norm.setdefault(norm(n), n)
    rep_names = report_names(report)
    fps = {os.path.splitext(f)[0] for f in os.listdir(fp_dir) if f.endswith(".kicad_mod")}
    groups = {}
    for key, p in distinct_parts(cfg_parts).items():
        if not p.get("wiki") or p.get("noModelExpected"):
            continue
        name = symbol_of(p, syms, by_norm, rep_names)
        fp = sym_property(sym_text, syms, name, "Footprint").split(":")[-1] if name else ""
        if not fp or fp not in fps:
            continue  # completeness names it
        refs, board_only = footprint_models(open(os.path.join(fp_dir, fp + ".kicad_mod")).read())
        if not refs and not board_only:
            groups.setdefault(fp, []).append((key, p))
    if not groups:
        return out
    tmp = tempfile.mkdtemp(prefix="aiflow-pagestep-")
    try:
        for fp, members in groups.items():
            found, whys = [], {}
            for key, p in members:
                pj, why = read_page_json(p["wiki"])
                f, rec, why2 = page_step_name(pj) if pj is not None else (None, None, f"its page.json could not be read ({why})")
                if not f:
                    whys[key] = why2; continue
                dest = os.path.join(tmp, re.sub(r"[^A-Za-z0-9_.-]", "_", p["wiki"]) + "__" + os.path.basename(f))
                got_file, detail = fetch_page_file(p["wiki"], f, dest)
                if not got_file:
                    whys[key] = f"its page names {f}, but that file could not be read from the page ({detail})"; continue
                if not is_step_file(dest):
                    whys[key] = f"its page's {f} is not a STEP file"; continue
                digest = sha(dest)
                if rec and digest != rec:
                    whys[key] = f"its page's {f} (sha256 {digest[:12]}) is not the file its page.json records (sha256 {rec[:12]})"; continue
                found.append({"key": key, "part": p, "file": f, "path": dest, "sha256": digest, "recorded": bool(rec)})
            if not whys and len({x["sha256"] for x in found}) > 1:
                for x in found:
                    whys[x["key"]] = f"footprint {fp} is shared by parts whose pages name different STEP models (" + ", ".join(f"{y['part']['mpn']}: {y['file']}" for y in found) + ")"
            if whys:
                for x in found:
                    whys.setdefault(x["key"], f"footprint {fp} is shared with a part whose page model is not bound")
                out["notBound"].update(whys)
                continue
            model = os.path.basename(found[0]["file"])
            target_file = os.path.join(m_dir, model)
            export_copy = "absent"
            if os.path.isfile(target_file):
                if sha(target_file) == found[0]["sha256"]:
                    export_copy = "same"
                else:
                    users = [g for g in os.listdir(fp_dir) if g.endswith(".kicad_mod") and model in footprint_models(open(os.path.join(fp_dir, g)).read())[0]]
                    if users:
                        for x in found:
                            out["notBound"][x["key"]] = f"another footprint ({', '.join(users)}) already uses a different model named {model}"
                        continue
                    export_copy = "replaced"
            if export_copy != "same":
                shutil.copy(found[0]["path"], target_file)
            fpath = os.path.join(fp_dir, fp + ".kicad_mod")
            target = f'${{KICAD10_3RD_PARTY}}/3dmodels/{pkgid.replace(".", "_")}/{lib}.3dshapes/{model}'
            text = add_model_line(open(fpath).read(), target)
            open(fpath, "w").write(text)
            for x in found:
                p = x["part"]
                out["bound"].append({"mpn": p["mpn"], "refs": p.get("refs") or [], "page": p["wiki"], "footprint": fp, "model": model,
                                     "pageFile": x["file"], "sha256": x["sha256"], "shaRecordedOnPage": x["recorded"], "exportCopy": export_copy})
    finally:
        shutil.rmtree(tmp, ignore_errors=True)
    return out


def completeness(cfg_parts, report, sym_path, fp_dir, m_dir, not_bound=None):
    """Every distinct BOM part resolves in the packaged library (issue #76): its symbol is there, the
    symbol's Footprint property names a footprint file in the export, and that footprint's (model ...)
    names a model file in the export (unless the part says noModelExpected, or the footprint is
    board_only). Parts may share one footprint or one model: file counts are reported, never compared.
    `not_bound` gives the reason a footprint with no model could not take its page's STEP (bind_page_models).
    Returns (counts, problems), the problems naming each part by MPN and references."""
    wiki = list(dict.fromkeys(p["wiki"] for p in cfg_parts if p.get("wiki")))
    proj = [p for p in cfg_parts if p.get("projectLib") and not p.get("wiki")]
    first = distinct_parts(cfg_parts)
    not_bound = not_bound or {}
    sym_text = open(sym_path).read() if os.path.isfile(sym_path) else ""
    syms = top_symbols(sym_text) if sym_text else {}
    by_norm = {}
    for n in syms:
        by_norm.setdefault(norm(n), n)
    fps = {os.path.splitext(f)[0] for f in os.listdir(fp_dir) if f.endswith(".kicad_mod")} if os.path.isdir(fp_dir) else set()
    models = [f for f in os.listdir(m_dir) if f.lower().endswith((".step", ".stp", ".wrl"))] if os.path.isdir(m_dir) else []
    model_set = set(models)
    distinct = len(wiki) + len({p["mpn"] for p in proj})
    rep = payload(report, "report") or report or {}
    rep_parts = payload(rep, "parts") or []
    skipped = payload(rep, "skipped") or []
    # the export report's own name for each MPN (the symbol may be named for the page, not the MPN)
    rep_names = {}
    for x in rep_parts:
        if isinstance(x, dict) and x.get("mpn") and x.get("name"):
            rep_names.setdefault(norm(x["mpn"]), []).append(x["name"])
    fp_models = {}

    def models_of(fp):
        if fp not in fp_models:
            t = open(os.path.join(fp_dir, fp + ".kicad_mod")).read()
            refs = [m.group(1) or m.group(2) for m in re.finditer(r'\(model\s+(?:"([^"]*)"|([^\s()]+))', t)]
            fp_models[fp] = ([os.path.basename(r.replace("\\", "/")) for r in refs], bool(re.search(r"\(attr\b[^)]*\bboard_only\b", t)))
        return fp_models[fp]

    missing, no_fp, no_model = [], [], []
    for key, p in first.items():
        label = p["mpn"] + (f" ({', '.join(p['refs'])})" if p.get("refs") else "")
        if p.get("projectLib") and not p.get("wiki"):
            name = p["projectLib"]["symbol"] if p["projectLib"]["symbol"] in syms else None
        else:
            name = next((by_norm[norm(c)] for c in mpn_candidates(p) + rep_names.get(norm(p["mpn"]), []) if norm(c) in by_norm), None)
        if not name:
            missing.append(label)
            continue
        fp = sym_property(sym_text, syms, name, "Footprint").split(":")[-1]
        if not fp or fp not in fps:
            no_fp.append(f"{label}: symbol {name} -> " + (f"footprint {fp} is not in the library" if fp else "no Footprint property"))
            continue
        refs, board_only = models_of(fp)
        if p.get("noModelExpected") or board_only:
            continue
        if not refs:
            no_model.append(f"{label}: footprint {fp} names no 3D model" + (f" (not bound from its page: {not_bound[key]})" if key in not_bound else ""))
        else:
            gone = [r for r in refs if r not in model_set]
            if gone:
                no_model.append(f"{label}: footprint {fp} -> model {', '.join(gone)} is not in the library")
    problems = []
    if missing:
        problems.append(f"{len(missing)} of {distinct} parts are not in the exported library: " + ", ".join(sorted(set(missing))))
    if skipped:
        problems.append("the export skipped: " + "; ".join(f"{x.get('ref') or x.get('name') or x} ({x.get('reason', '')})" if isinstance(x, dict) else str(x) for x in skipped))
    if no_fp:
        problems.append("parts whose footprint does not resolve: " + "; ".join(no_fp[:12]) + (f" (and {len(no_fp) - 12} more)" if len(no_fp) > 12 else ""))
    if no_model:
        problems.append("parts whose 3D model does not resolve: " + "; ".join(no_model[:12]) + (f" (and {len(no_model) - 12} more)" if len(no_model) > 12 else ""))
    counts = {"distinctParts": distinct, "symbols": len(syms), "footprints": len(fps), "models": len(models)}
    return counts, problems


def pcm_content_sha(stage):
    """The package's content: every file but metadata.json, by path and bytes (never the zip, whose
    timestamps change on every build)."""
    h = hashlib.sha256()
    for d, dirs, fs in os.walk(stage):
        dirs.sort()
        for f in sorted(fs):
            p = os.path.join(d, f); rel = os.path.relpath(p, stage).replace(os.sep, "/")
            if rel == "metadata.json":
                continue
            h.update(rel.encode() + b"\0"); h.update(open(p, "rb").read()); h.update(b"\0")
    return h.hexdigest()


def vtuple(v):
    return tuple(int(x) if x.isdigit() else 0 for x in re.split(r"[.\-+]", str(v or "0")))


def pcm_version(previous, content, base="1.0.0"):
    """The package version for this content (issue #169). KiCad's PCM installs a version once and updates
    only to a higher one, so a rebuild whose content differs from the last build gets the next patch
    version; the same content keeps its version. previous: [{"version", "contentSha256"?}] from the run's
    own record and the page's published pcm files; an entry without a hash counts as different content."""
    prev = [p for p in previous or [] if p and p.get("version")]
    if not prev:
        return base, "first build"
    top = max(prev, key=lambda p: (vtuple(p["version"]), bool(p.get("contentSha256"))))
    if top.get("contentSha256") == content:
        return top["version"], f"unchanged since {top['version']}"
    parts = [int(x) if x.isdigit() else 0 for x in str(top["version"]).split("+")[0].split("-")[0].split(".")]
    parts = (parts + [0, 0, 0])[:max(3, len(parts))]; parts[-1] += 1
    bumped = ".".join(map(str, parts))
    return max(bumped, base, key=vtuple), f"content changed since {top['version']}"


def pcm_previous(cfg, folder, work, pkgid):
    """The package versions built before: the run's own pcm/build.json, then (unless local only) the
    page's published pcm/build.json and pcm/packages.json. A failed read is no previous build."""
    out = []
    try:
        b = json.load(open(os.path.join(work, "pcm", "build.json")))
        if b.get("identifier") == pkgid: out.append(b)
    except (OSError, ValueError):
        pass
    if cfg.get("localOnly") or not (cfg.get("push") or cfg.get("existing") or folder):
        return out
    for name in ("build.json", "packages.json"):
        p = subprocess.run(["adom-wiki", "repo", "show", cfg["page"], under(folder, f"pcm/{name}")], capture_output=True, text=True, stdin=subprocess.DEVNULL)
        if p.returncode != 0 or "{" not in p.stdout:
            continue
        try:
            v = json.loads(p.stdout[p.stdout.index("{"):p.stdout.rindex("}") + 1])
        except ValueError:
            continue
        if name == "build.json":
            if v.get("identifier") == pkgid: out.append(v)
        else:
            out += [{"version": x.get("version")} for pk in v.get("packages") or [] if pk.get("identifier") == pkgid for x in pk.get("versions") or []]
    return out


def pcm_package(bundle, lib, pkgid, version, title, brief, page_url, work, parts=(), report=None):
    """Repackage the Cloud Library bundle (and the project-library parts) in the PCM layout, bind a wiki part's
    own page STEP where its footprint names no model (bind_page_models), normalise and parse-check the symbol
    library; returns (zip path, install size, metadata, check)."""
    src = os.path.join(work, "bundle"); shutil.rmtree(src, ignore_errors=True); os.makedirs(src)
    if bundle:
        with zipfile.ZipFile(bundle) as z: z.extractall(src)
    root = next((os.path.join(src, d) for d in os.listdir(src) if os.path.isdir(os.path.join(src, d))), src)
    stage = os.path.join(work, "pcm-stage"); shutil.rmtree(stage, ignore_errors=True)
    sym_dir, fp_dir, m_dir = (os.path.join(stage, "symbols"), os.path.join(stage, "footprints", f"{lib}.pretty"), os.path.join(stage, "3dmodels", f"{lib}.3dshapes"))
    for d in (sym_dir, fp_dir, m_dir): os.makedirs(d)
    models = os.path.join(root, "3dmodels")
    for f in os.listdir(models) if os.path.isdir(models) else []:
        shutil.copy(os.path.join(models, f), m_dir)
    for f in os.listdir(root):
        if f.endswith(".kicad_sym"): shutil.copy(os.path.join(root, f), os.path.join(sym_dir, f"{lib}.kicad_sym"))
        if f.endswith(".pretty"):
            for m in os.listdir(os.path.join(root, f)):
                t = open(os.path.join(root, f, m)).read()
                # bundle-relative 3D paths -> where PCM installs them
                t = re.sub(r'\(model\s+"?3dmodels/([^")\s]+)"?', lambda mm: f'(model "${{KICAD10_3RD_PARTY}}/3dmodels/{pkgid.replace(".", "_")}/{lib}.3dshapes/{mm.group(1)}"', t)
                open(os.path.join(fp_dir, m), "w").write(t)
    sp = os.path.join(sym_dir, f"{lib}.kicad_sym")
    t = quote_atoms(open(sp).read()) if os.path.isfile(sp) else ""
    # one entry per MPN: the parts record lists a part once per reference group, so a value on several
    # references (four 1 uF caps) arrives more than once and must be merged only once
    proj = list({p["mpn"]: p for p in reversed([p for p in parts if p.get("projectLib") and not p.get("wiki")])}.values())[::-1]
    versions = [sym_header_version(t) or 0] + [sym_header_version(open(p["projectLib"]["symbolFile"]).read()) or 0 for p in proj]
    t = merge_project_parts(proj, t, fp_dir, m_dir, lib, pkgid)
    if t:
        # one header for every symbol: the newest syntax any of them uses, at least KiCad 9's
        header = max([SYM_VERSION] + versions)
        t = normalise_sym_header(t, header)
        t = unit_names_follow_parent(t)
        # the symbols point at the footprints by library nickname: PCM installs this library as PCM_<lib>
        t = re.sub(r'(\(property\s+"Footprint"\s+")([^":]*):', lambda mm: f'{mm.group(1)}PCM_{lib}:', t)
        open(sp, "w").write(t)
    else:
        header = SYM_VERSION
    page_models = bind_page_models(parts, report, sp, fp_dir, m_dir, lib, pkgid)
    problems = structural_check(sp, fp_dir) if os.path.isfile(sp) else ["no symbol library"]
    kstatus, kdetail = kicad_check(sp) if not problems else ("skipped", "the structural parse failed")
    broken = []
    if kstatus == "fail":
        broken = broken_symbols(sp, parts)
        if broken:
            problems.append("; ".join(f"{b['label']}: KiCad cannot load this symbol" for b in broken) +
                            ". Fix or replace that part's symbol on its wiki page, or change the part, then run project-page again")
        else:
            problems.append("KiCad cannot load the packaged symbol library, though each symbol loads alone: " + kdetail)
    check = {"symVersion": header, "structural": "pass" if not problems else "fail", "kicadCli": kstatus, "kicadCliDetail": kdetail, "problems": problems, "brokenSymbols": broken,
             "pageModels": page_models}
    if problems:
        fail("the packaged KiCad library does not load: " + "; ".join(problems), libraryCheck=check)
    has_wiki = any(p.get("wiki") for p in parts)
    meta = {"$schema": "https://go.kicad.org/pcm/schemas/v1", "name": title, "description": short(brief)[:150],
            "description_full": f"{brief}\n\nEvery part of this board" + (", from its Adom wiki component page" if has_wiki and not proj else (" (from the Adom wiki and the board's project library)" if has_wiki else ", from the board's project library")) +
                                f": symbol, footprint and STEP model. Project page: {page_url}",
            "identifier": pkgid, "type": "library", "author": {"name": "Adom wiki", "contact": {"web": page_url}},
            "license": "MIT", "resources": {"homepage": page_url},
            "versions": [{"version": None, "status": "stable", "kicad_version": "10.0" if header >= KICAD10_SYM_VERSION else "9.0"}]}
    # issue #169: the version follows the content, so KiCad's PCM sees a changed library as an update
    content = pcm_content_sha(stage)
    version, why = version(content) if callable(version) else (version, "given")
    meta["versions"][0]["version"] = version
    check["pcm"] = {"version": version, "contentSha256": content, "why": why}
    json.dump(meta, open(os.path.join(stage, "metadata.json"), "w"), indent=1)
    out = os.path.join(work, "pcm", f"{lib}-pcm-{version}.zip"); os.makedirs(os.path.dirname(out), exist_ok=True)
    size = 0
    with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as z:
        for d, _, fs in os.walk(stage):
            for f in sorted(fs):
                p = os.path.join(d, f); z.write(p, os.path.relpath(p, stage)); size += os.path.getsize(p)
    check["paths"] = {"symbols": sp, "footprints": fp_dir, "models": m_dir}
    return out, size, meta, check


def short(brief):
    """The first line of prose in a brief (a markdown file's heading and blank lines skipped)."""
    for line in (brief or "").splitlines():
        if line.lstrip().startswith("#"):
            continue
        line = line.strip().lstrip(">*- ").strip()
        if line:
            return line
    return brief or ""


def altium_section(parts):
    """The README's Altium section: where the SchLib and PcbLib come from (the wiki pages, and the board's
    own project library for parts with no public page yet, converted the same way)."""
    proj = sorted({p["mpn"] for p in parts if p.get("projectLib") and not p.get("wiki")})
    src = "the parts' wiki pages above" + (f" and, for {', '.join(proj)} (no public page yet), from the board's own project library: the same KiCad symbol, footprint and STEP model converted the same way" if proj else "")
    return f"""## Built in Altium Designer

AI Flow builds this board's Altium libraries from {src}: one schematic library (SchLib) and one PCB library (PcbLib) with every symbol, footprint and 3D body, the part numbers and every distributor number on each symbol. The schematic and the board are drawn with them in Altium Designer through the Adom Altium Bridge.
"""


def molecule_note(cfg):
    """A molecule page's 3D model, footprint and symbol come from the wiki's model registry (issue #137)."""
    if not cfg.get("molecule"):
        return ""
    return (" This board is an Adom molecule: once it is finished its sources (STEP, board, schematic and fabrication files)"
            " are published here with [molecule-publish](https://wiki.adom.inc/adom/molecule-publish), and the wiki builds its"
            " 3D model, footprint and symbol from them.")


def readme(cfg, lib, pkgid, repo_url):
    rows, merged = [], {}
    for p in cfg["parts"]:  # one row per MPN, its references together
        m = merged.setdefault(p["mpn"], dict(p, refs=[]))
        m["refs"] = m["refs"] + list(p.get("refs") or [])
    for p in merged.values():
        refs = ", ".join(p.get("refs") or [])
        wiki = f"[{p['wiki']}](https://wiki.adom.inc/{p['wiki']})" if p.get("wiki") else ("project library (no public page yet)" if p.get("projectLib") else "not on the wiki yet")
        rows.append(f"| {refs} | {len(p.get('refs') or [])} | {p.get('value', '')} | {p['mpn']} | {p.get('manufacturer', '')} | {wiki} |")
    return f"""# {cfg['title']}

{cfg.get('summary') or re.sub(r"\A\s*#[^\n]*\n+", "", cfg['brief'])}

**Status: revision 1.** The parts are chosen and every one has its Adom wiki page{" or comes from the board's own project library" if any(p.get("projectLib") and not p.get("wiki") for p in cfg["parts"]) else ""}; the schematic and the board come next, built with [adom-aiflow](https://wiki.adom.inc/adom/adom-aiflow){" in Altium Designer" if cfg.get("eda") == "altium" else ""}.{molecule_note(cfg)}

## Components

| References | Qty | Value | MPN | Manufacturer | Wiki page |
|---|---|---|---|---|---|
{chr(10).join(rows)}

{altium_section(cfg["parts"]) if cfg.get("eda") == "altium" else ""}{"" if cfg.get("eda") == "altium" else f"""## Install every part in KiCad

This page is a KiCad Plugin and Content Manager repository. In KiCad, open **Plugin and Content Manager**, **Manage...**, add this URL, then install **{cfg['title']}** from the Libraries tab:

```
{repo_url}
```

The library appears as `PCM_{lib}` (package `{pkgid}`): every symbol, footprint and 3D model of this board, from the parts' wiki pages.
"""}"""


def main():
    cfg = json.load(open(sys.argv[1]))
    if cfg.get("localOnly"):
        cfg["push"] = False   # local only implies no push (issue #157)
    owner, slug = cfg["page"].split("/", 1)
    work = os.path.join(cfg["run"], "project-page"); os.makedirs(work, exist_ok=True)
    folder = check_dir(cfg.get("pageDir"))
    page_url = f"https://wiki.adom.inc/{owner}/{slug}" + (f"/files/{folder}/README.md" if folder else "")
    created = False
    exists = cfg.get("push") and page_exists(cfg["page"])
    if folder and (cfg.get("create") or (cfg.get("push") and not exists)):
        # a folder of a page: the page is the project's and must exist; this never makes one
        raise SystemExit(json.dumps({"ok": False, "error": f"{cfg['page']} " + ("does not exist" if not cfg.get("create") else "is named with --create") + f": --page-dir {folder} writes into a folder of an existing page and never creates one"}))
    # a page this run created and has not finished (issue #125): a rerun carries on, README included
    made_rec = os.path.join(work, "page-created.json")
    try:
        unfinished = json.load(open(made_rec)).get("page") == cfg["page"]
    except (OSError, ValueError):
        unfinished = False
    if exists and unfinished:
        created = True
    if exists and not cfg.get("existing") and not folder and not unfinished:
        # a new board gets its own new page: never add one board's library to another board's (or anyone's) page
        raise SystemExit(json.dumps({"ok": False, "error": f"{cfg['page']} already exists. A new board gets its own new page: choose a new slug (--create). Pass --existing only when that page is this same board's own page from an earlier run."}))
    if cfg.get("push") and not exists and not cfg.get("create"):  # a dry run (--no-push) builds everything locally
        raise SystemExit(json.dumps({"ok": False, "error": f"{cfg['page']} does not exist; pass --create to make it"}))

    kind = "molecule" if cfg.get("molecule") else "board"

    def make_page():
        """The page is made only once there is something to put on it, so a failed library check leaves none."""
        nonlocal created
        if not cfg.get("push") or exists:
            return
        # issue #137: a molecule (the spec has "molecule") gets a page of kind molecule, which the wiki's model
        # registry builds the GLB, footprint and symbol for from the sources molecule-publish puts there
        args = ["adom-wiki", "page", "create", slug, "--type", "hardware", "--kind", kind, "--title", cfg["title"], "--brief", cfg["brief"][:200],
                "--because", f"a new {'molecule' if kind == 'molecule' else 'board'} designed by adom-aiflow: its own project page, one page per board",
                "--headline", cfg["title"][:60], "--subhead", cfg["brief"][:100], "--source", "altium" if cfg.get("eda") == "altium" else "kicad"]
        me = (wjson(["adom-wiki", "whoami"]).get("data") or {}).get("user", {}).get("username") or ""
        if owner != me: args += ["--org", owner]
        if cfg.get("private"): args.append("--private")
        sh(args); created = True
        json.dump({"page": cfg["page"]}, open(made_rec, "w"))

    def page_finished():
        if os.path.exists(made_rec):
            os.remove(made_rec)

    lib, pkgid = (lib_name(f"{slug}-{folder.rsplit('/', 1)[-1]}"), pkg_id(owner, f"{slug}.{folder}")) if folder else (lib_name(slug), pkg_id(owner, slug))
    version = cfg.get("version") or "1.0.0"
    if cfg.get("eda") == "altium":
        # Altium: the library step builds the SchLib and PcbLib from the same wiki pages; no KiCad PCM package
        open(os.path.join(work, "README.md"), "w").write(readme(cfg, lib, pkgid, None))
        pushed = None
        make_page()
        if cfg.get("push") and readme_wanted(cfg, folder, created):
            pushed = push_files(cfg, work, folder, ["README.md"], "Revision 1: the chosen parts and the component table (adom-aiflow, Altium Designer)")
        page_finished()
        print(json.dumps({"ok": True, "eda": "altium", "kind": kind, "page": cfg["page"], "pageDir": folder, "url": page_url, "created": created, "cloudLibrary": None, "libName": lib,
                          "identifier": None, "repositoryUrl": None, "parts": len(cfg["parts"]), "counts": {}, "libraryCheck": {},
                          "adds": {"addedPages": 0, "rateLimitWaits": [], "notes": []}, "projectLibParts": [p["mpn"] for p in cfg["parts"] if p.get("projectLib") and not p.get("wiki")],
                          "readme": os.path.join(work, "README.md"), "commit": pushed}))
        return
    # issue #157: --local-only makes no wiki call at all. The wiki parts' files come only from a Cloud Library
    # export (library create/add/export are remote calls), so a local-only build packages the project-library
    # parts and names the wiki parts it left out.
    local = bool(cfg.get("localOnly"))
    pkg_parts = [p for p in cfg["parts"] if not (local and p.get("wiki"))]
    left_out = [f"{p['mpn']} ({p['wiki']})" for p in cfg["parts"] if local and p.get("wiki")]
    if any(p.get("wiki") for p in pkg_parts):
        lib_ref, bundle, report, adds = cloud_library(f"{owner}/{slug}/{folder}" if folder else f"{owner}/{slug}", pkg_parts, work, cfg.get("libRef"))
    else:   # every part from the project library: no Cloud Library needed
        lib_ref, bundle, report, adds = None, None, {}, {"addedPages": 0, "rateLimitWaits": [], "notes": []}
    if local and not any(p.get("projectLib") for p in pkg_parts):
        open(os.path.join(work, "README.md"), "w").write(readme(cfg, lib, pkgid, None))
        print(json.dumps({"ok": True, "kind": kind, "localOnly": True, "page": cfg["page"], "pageDir": folder, "url": page_url, "created": False, "cloudLibrary": None, "libName": lib,
                          "identifier": None, "repositoryUrl": None, "zip": None, "parts": len(cfg["parts"]), "counts": {}, "libraryCheck": {}, "localOnlyLeftOut": left_out,
                          "adds": adds, "projectLibParts": [], "readme": os.path.join(work, "README.md"), "commit": None}))
        return
    previous = pcm_previous(cfg, folder, work, pkgid)
    zpath, isize, meta, check = pcm_package(bundle, lib, pkgid, lambda c: pcm_version(previous, c, version), cfg["title"], cfg["brief"], page_url, work, pkg_parts, report)
    counts, gaps = completeness(pkg_parts, report, check["paths"]["symbols"], check["paths"]["footprints"], check["paths"]["models"],
                                (check.get("pageModels") or {}).get("notBound"))
    if gaps:
        # never publish a library that is short of the board's parts
        fail("the KiCad library is incomplete: " + "; ".join(gaps) + (f" (Cloud Library {lib_ref})" if lib_ref else ""),
             cloudLibrary=lib_ref, counts=counts, libraryCheck=check, pageModels=check.get("pageModels"))
    zname = os.path.basename(zpath)
    files_url = f"{API}/{owner}/{slug}/files" + (f"/{folder}" if folder else "")
    v = dict(meta["versions"][0]); v.update({"download_url": f"{files_url}/pcm/{zname}", "download_sha256": sha(zpath), "download_size": os.path.getsize(zpath), "install_size": isize})
    pkgs = {"$schema": "https://go.kicad.org/pcm/schemas/v1#/definitions/PackageArray", "packages": [dict(meta, versions=[v])]}
    pj = os.path.join(work, "pcm", "packages.json"); json.dump(pkgs, open(pj, "w"), indent=1)
    now = int(time.time())
    repo = {"$schema": "https://go.kicad.org/pcm/schemas/v1#/definitions/Repository", "name": f"{cfg['title']} (Adom wiki)",
            "maintainer": {"name": owner, "contact": {"web": page_url}},
            "packages": {"url": f"{files_url}/pcm/packages.json", "sha256": sha(pj), "update_timestamp": now, "update_time_utc": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now))}}
    rp = os.path.join(work, "pcm", "repository.json"); json.dump(repo, open(rp, "w"), indent=1)
    json.dump(dict(check["pcm"], identifier=pkgid, zip=zname), open(os.path.join(work, "pcm", "build.json"), "w"), indent=1)
    repo_url = f"{files_url}/pcm/repository.json"
    open(os.path.join(work, "README.md"), "w").write(readme(cfg, lib, pkgid, repo_url))
    pushed = None
    make_page()
    if cfg.get("push"):
        files = [f"pcm/{zname}", "pcm/packages.json", "pcm/repository.json", "pcm/build.json"]
        if readme_wanted(cfg, folder, created): files.insert(0, "README.md")
        pushed = push_files(cfg, work, folder, files, "Revision 1: the chosen parts, the component table and the KiCad PCM repository (adom-aiflow)")
    page_finished()
    print(json.dumps({"ok": True, "kind": kind, "localOnly": local, "localOnlyLeftOut": left_out, "page": cfg["page"], "pageDir": folder, "url": page_url, "created": created, "cloudLibrary": lib_ref, "libName": lib,
                      "identifier": pkgid, "repositoryUrl": repo_url, "zip": zpath, "parts": len(cfg["parts"]), "counts": counts, "libraryCheck": check,
                      "adds": adds, "projectLibParts": [p["mpn"] for p in cfg["parts"] if p.get("projectLib") and not p.get("wiki")],
                      "pageModels": check.get("pageModels"), "pcmVersion": check["pcm"]["version"], "pcmVersionWhy": check["pcm"]["why"],
                      "exportReport": payload(report, "report"),
                      "readme": os.path.join(work, "README.md"), "commit": pushed}))


if __name__ == "__main__":
    main()