molecule-publish

--- a/.pkgignore+++ b/.pkgignore@@ -0,0 +1,4 @@+__pycache__/
+*.pyc
+tests/+--- a/docs/fabrication.md+++ b/docs/fabrication.md@@ -0,0 +1,74 @@+# Fabrication sources++Every new KiCad, Fusion or Altium molecule export includes `<slug>-fab.zip`+and `export.json`. Make them from the same saved board revision as the STEP.+Finish source edits, including custom-model path rewrites, before exporting.+Use a fresh folder per export so old files cannot enter a new bundle.++The zip contains both sides' copper, solder mask and silkscreen, the actual+board outline, and Excellon drill files. Keep plated and unplated drills+separate where the exporter supports it; record mixed drills honestly.+Keep the original file bytes and names inside the zip. Optional inner+copper, paste and Gerber job files may travel too. Preserve silk PNGs while+the current converter still consumes them.++`export.json` uses the existing `adom.export/1` contract from Fusion PR #17:++- `tool`, `toolVersion`, `exporter`, `exporterVersion`: actual producers.+- `files[]`: relative path, role, byte count and SHA-256 for the STEP, saved+  board, schematic sheets and every additional source handed to publishing.+- The `fab_zip` file has `entries[]`, each with its relative path inside+  the zip, role, byte count and SHA-256. Nested entries avoid confusing+  archive members with real page paths.+- Layer roles: `copper_top`, `copper_bottom`, `mask_top`, `mask_bottom`,+  `silk_top`, `silk_bottom`, `outline`; drill roles: `drill_plated`,+  `drill_unplated`, `drill_mixed`; optional `paste_top`, `paste_bottom`,+  `copper_inner_1` (and subsequent inner layers), `gerber_job` and `other`.+- Source roles: `step`, `board_kicad_pcb`, `board_eagle_brd`,+  `board_altium_pcbdoc`, `schematic`, `project`, and descriptive roles for+  optional sources (for example `fusion_archive`, `usdz`, `bom`, `cpl`).+- `stackup`: known `thicknessMm`, `copperLayers`, `maskColor` and+  `silkColor` by `top`/`bottom`, and `finish`. Unknown values stay null or+  absent; never turn a viewer's default into claimed design data.+- `gerbers.included` is true. `drill.plating` says `split` or `mixed`.+  Exporters may also preserve `origin`, `outline`, `exportedAt` and their+  own provenance. A historical backfill carries `backfill` and its original+  `tier`; it must never claim to be a fresh native EDA export.++The helper in this package is Python 3.9+ with no third-party dependencies.+Run it from the package directory, with the export folder as its argument:++```sh+python3 scripts/fab_bundle.py create /path/to/bundle --slug my-board \+  --fab /path/to/fresh-gerbers --tool KiCad --tool-version 10.0.3 \+  --exporter adom/kicad-export-molecule --exporter-version 1.2.0 \+  --source step=Board.step --source board_kicad_pcb=Board.kicad_pcb \+  --source schematic=Board.kicad_sch --source project=Board.kicad_pro \+  --stackup /path/to/known-stackup.json+python3 scripts/fab_bundle.py check /path/to/bundle --slug my-board+```++Use the versions actually installed, not the example values. `--fab` also+accepts an EDA-produced zip. `--stackup` is optional; pass the data the EDA+provides. Repeat `--source` for every exported source, including every+schematic sheet. The helper writes the canonical zip name without changing+its members' bytes. It refuses missing layers, a blank outline, missing drill+data and changed hashes. It does not prove alignment or validate the circuit;+the existing conversion and manufacturing checks still apply.++For a bridge-produced record, verify it before any packaging rename. If the+bridge used a design stem instead of the wiki slug, use a fresh staging+folder and regenerate the record against the final filenames. Never rename+a recorded source or zip without updating and checking the record.++Before publishing, run `check` after all local source changes. Include the+zip, record and every recorded source in both the package and the explicit+repo push list. Re-fetch those files at the returned commit into a fresh+folder, then run the same check there. A successful push response alone does+not prove that the complete export reached the page.++This contract applies to EDA molecule exports. tscircuit retains+`circuit.json` and its build inputs; its pipeline generates fabrication+layers. Existing pages are backfilled separately. This source addition does+not enable registry model serving or replace any page's current model.+--- a/install.sh+++ b/install.sh@@ -1,9 +1,13 @@⋯ 5 unchanged lines ⋯ rm -rf "$SKILLS_DIR/molecule-publish" mkdir -p "$SKILLS_DIR/molecule-publish" cp "$PKG_DIR/SKILL.md" "$SKILLS_DIR/molecule-publish/SKILL.md"+cp -R "$PKG_DIR/scripts" "$SKILLS_DIR/molecule-publish/scripts"+mkdir -p "$SKILLS_DIR/molecule-publish/docs"+cp "$PKG_DIR/docs/fabrication.md" "$SKILLS_DIR/molecule-publish/docs/fabrication.md" echo "Installed skill → $SKILLS_DIR/molecule-publish/SKILL.md"+--- a/package.json+++ b/package.json@@ -1,47 +1,48 @@ {   "slug": "molecule-publish",-  "version": "2.10.0",+  "version": "2.11.0",   "type": "skill",   "description": "Publish molecules AND generic boards to the Adom wiki - one publisher, two shapes: OCCT conversion (molecule mode with stats gates, or plain for boards), component packaging, adom-wiki publish, APM render assets + component dependencies. Companion to adom-molecule-import.",   "tags": [⋯ 38 unchanged lines ⋯     }   ] }+--- a/page.json+++ b/page.json@@ -1,54 +1,54 @@ {   "slug": "molecule-publish",-  "version": "2.10.0",+  "version": "2.11.0",   "type": "skill",   "description": "Publish molecules AND generic boards to the Adom wiki - one publisher, two shapes: OCCT conversion (molecule mode with stats gates, or plain for boards), component packaging, adom-wiki publish, APM render assets + component dependencies. Companion to adom-molecule-import.",   "tags": [⋯ 48 unchanged lines ⋯--- a/README.md+++ b/README.md@@ -1,21 +1,23 @@⋯ 17 unchanged lines ⋯ | yes | STEP (`.step`) | exported ON the design machine so all 3D models (machine pins, contacts, MP markers) are baked in | | yes | Board/layout (`.kicad_pcb` or `.brd`) | what footprint+symbol generation parses | | yes | Schematic (`.kicad_sch` or `.sch`) | source completeness — a molecule without its schematic is not reproducible |+| EDA molecule | `<slug>-fab.zip` + `export.json` | Required fabrication layers, drill files and hashed export provenance; see [fabrication.md](docs/fabrication.md) | | yes | Metadata | `package.json` (authored below) + the OCCT-generated footprint/symbol JSONs |+--- a/scripts/fab_bundle.py+++ b/scripts/fab_bundle.py@@ -0,0 +1,218 @@+#!/usr/bin/env python3+"""Pack and verify fabrication sources. Standard library only; never publishes."""+import argparse+import hashlib+import json+import re+import sys+import zipfile+from pathlib import Path, PurePosixPath++VERSION = "1.0.0"+REQUIRED = {"copper_top", "copper_bottom", "mask_top", "mask_bottom",+            "silk_top", "silk_bottom", "outline"}+DRILLS = {"drill_plated", "drill_unplated", "drill_mixed"}+EXT_ROLES = {"gtl": "copper_top", "gbl": "copper_bottom", "gts": "mask_top",+             "gbs": "mask_bottom", "gto": "silk_top", "gbo": "silk_bottom",+             "gtp": "paste_top", "gbp": "paste_bottom", "gko": "outline",+             "gm1": "outline", "gbrjob": "gerber_job"}+++def safe_path(name):+    p = PurePosixPath(name)+    if (not name or "\\" in name or ":" in name or p.is_absolute()+            or any(v in ("", ".", "..") for v in name.split("/"))):+        raise ValueError(f"not a relative file path: {name!r}")+    return p+++def fingerprint(name, role, data):+    return {"path": name, "role": role, "bytes": len(data),+            "sha256": hashlib.sha256(data).hexdigest()}+++def role_for(name, data):+    ext = Path(name).suffix.lower().lstrip(".")+    text = data.decode("utf-8", errors="replace")+    # X2 is authoritative, including KiCad's .gbr files and named imported layers.+    match = re.search(r"TF\.FileFunction,([^*%\r\n]+)", text)+    if match:+        parts = match[1].split(",")+        kind = parts[0]+        side = "top" if "Top" in parts else "bottom" if "Bot" in parts else None+        if kind == "Profile":+            return "outline"+        if kind in ("Copper", "Soldermask", "Legend", "Paste") and side:+            return {"Copper": "copper", "Soldermask": "mask", "Legend": "silk",+                    "Paste": "paste"}[kind] + "_" + side+        if kind == "Copper":+            layer = next((p for p in parts if re.fullmatch(r"L[2-9][0-9]*", p)), None)+            return "copper_inner_" + str(int(layer[1:]) - 1) if layer else "copper_inner"+    if ext in ("drl", "xln", "exc", "txt") and re.search(r"(?m)^M48\s*$", text):+        stem = Path(name).stem.upper()+        if "NPTH" in stem or "NONPLATED" in stem or "NonPlated" in text:+            return "drill_unplated"+        if "PTH" in stem or stem.endswith("-PLATED") or re.search(r"FileFunction,Plated,", text):+            return "drill_plated"+        return "drill_mixed"+    if re.fullmatch(r"g[1-9][0-9]*", ext):+        return "copper_inner_" + ext[1:]+    return EXT_ROLES.get(ext, "other")+++def read_zip(path):+    out = {}+    with zipfile.ZipFile(path) as z:+        for item in z.infolist():+            if item.is_dir():+                continue+            safe_path(item.filename)+            if item.filename in out:+                raise ValueError(f"duplicate zip entry: {item.filename}")+            out[item.filename] = z.read(item)+    return out+++def layer_entries(files):+    entries = [fingerprint(n, role_for(n, b), b) for n, b in sorted(files.items())]+    roles = {e["role"] for e in entries}+    missing = REQUIRED - roles+    if missing:+        raise ValueError("missing fabrication layers: " + ", ".join(sorted(missing)))+    if not roles & DRILLS:+        raise ValueError("missing Excellon drill file (M48 header)")+    # An empty mechanical-layer plot is not a board outline (Altium can emit one).+    outlines = [files[e["path"]].decode("utf-8", errors="replace")+                for e in entries if e["role"] == "outline"]+    if not any(re.search(r"[XY][+-]?\d+[^*]*D0?1\*", t) for t in outlines):+        raise ValueError("outline Gerber has no drawn segments; export the actual board boundary")+    return entries+++def source_entry(root, value):+    role, sep, name = value.partition("=")+    if not sep or not role:+        raise ValueError("source must be ROLE=relative/path")+    safe_path(name)+    file = (root / name).resolve()+    if not file.is_relative_to(root.resolve()):+        raise ValueError(f"source escapes bundle: {name}")+    return fingerprint(name, role, file.read_bytes())+++def check(root, slug):+    root = Path(root)+    rec = json.loads((root / "export.json").read_text(encoding="utf-8"))+    if rec.get("schema") != "adom.export/1":+        raise ValueError("export.json needs schema adom.export/1")+    for key in ("tool", "toolVersion", "exporter", "exporterVersion"):+        if not isinstance(rec.get(key), str) or not rec[key].strip():+            raise ValueError(f"export.json needs {key}")+    entries = rec.get("files", [])+    if not isinstance(entries, list) or not entries:+        raise ValueError("export.json needs files[]")+    names = set()+    for entry in entries:+        name = entry["path"]+        safe_path(name)+        if name in names:+            raise ValueError(f"duplicate file record: {name}")+        names.add(name)+        path = (root / name).resolve()+        if not path.is_relative_to(root.resolve()):+            raise ValueError(f"file escapes bundle: {name}")+        actual = fingerprint(name, entry["role"], path.read_bytes())+        if any(entry.get(k) != actual[k] for k in ("sha256", "bytes")):+            raise ValueError(f"source changed since export: {name}")+    roles = {e["role"] for e in entries}+    if not roles & {"board_kicad_pcb", "board_eagle_brd", "board_altium_pcbdoc"}:+        raise ValueError("export.json needs the board source and its hash")+    if not rec.get("backfill") and "step" not in roles:+        raise ValueError("export.json needs the STEP source and its hash")+    zips = [e for e in entries if e["role"] == "fab_zip"]+    if len(zips) != 1 or zips[0]["path"] != f"{slug}-fab.zip":+        raise ValueError(f"exactly one fab_zip named {slug}-fab.zip is required")+    files = read_zip(root / zips[0]["path"])+    actual = layer_entries(files)+    expected = zips[0].get("entries", [])+    by_name = {e["path"]: e for e in expected}+    if len(by_name) != len(expected) or set(by_name) != set(files):+        raise ValueError("zip entries and export.json disagree")+    for entry in actual:+        saved = by_name[entry["path"]]+        if any(saved.get(k) != entry[k] for k in ("sha256", "bytes", "role")):+            raise ValueError(f"zip entry changed or has wrong role: {entry['path']}")+    return rec+++def create(args):+    root = Path(args.root).resolve()+    if (root / "export.json").exists():+        raise ValueError("export.json already exists; verify it or use a fresh export folder")+    source = Path(args.fab)+    if source.is_dir():+        files = {}+        for p in sorted(source.rglob("*")):+            if p.is_symlink():+                raise ValueError(f"symlink in fabrication folder: {p}")+            if p.is_file():+                name = p.relative_to(source).as_posix()+                safe_path(name)+                files[name] = p.read_bytes()+    else:+        files = read_zip(source)+    layers = layer_entries(files)+    entries = [source_entry(root, s) for s in args.source]+    target = root / f"{args.slug}-fab.zip"+    if target.exists():+        raise ValueError(f"output exists: {target}; use a fresh export folder")+    # Fixed ZIP metadata; the Gerber/drill bytes remain exactly as exported.+    with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as z:+        for name, data in sorted(files.items()):+            item = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))+            item.compress_type = zipfile.ZIP_DEFLATED+            item.external_attr = 0o644 << 16+            z.writestr(item, data)+    fab = fingerprint(target.name, "fab_zip", target.read_bytes())+    fab["entries"] = layers+    entries.append(fab)+    stackup = json.loads(Path(args.stackup).read_text(encoding="utf-8")) if args.stackup else {}+    rec = {"schema": "adom.export/1", "tool": args.tool, "toolVersion": args.tool_version,+           "exporter": args.exporter, "exporterVersion": args.exporter_version,+           "design": args.slug, "files": entries, "stackup": stackup,+           "gerbers": {"included": True},+           "drill": {"plating": "mixed" if any(e["role"] == "drill_mixed" for e in layers) else "split"}}+    (root / "export.json").write_text(json.dumps(rec, indent=2) + "\n", encoding="utf-8")+    return check(root, args.slug)+++def main():+    p = argparse.ArgumentParser(description=__doc__)+    sub = p.add_subparsers(dest="command", required=True)+    for verb in ("create", "check"):+        q = sub.add_parser(verb)+        q.add_argument("root", help="export folder; all recorded source paths are relative to it")+        q.add_argument("--slug", required=True)+        if verb == "create":+            q.add_argument("--fab", required=True, help="EDA fabrication output directory or zip")+            q.add_argument("--tool", required=True)+            q.add_argument("--tool-version", required=True)+            q.add_argument("--exporter", required=True)+            q.add_argument("--exporter-version", required=True)+            q.add_argument("--source", action="append", required=True, help="ROLE=relative/path (repeat)")+            q.add_argument("--stackup", help="JSON with known thicknessMm, maskColor, silkColor, finish")+    args = p.parse_args()+    try:+        if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", args.slug):+            raise ValueError("slug must be lowercase words separated by hyphens")+        rec = create(args) if args.command == "create" else check(args.root, args.slug)+        print(json.dumps({"ok": True, "files": [e["path"] for e in rec["files"]] + ["export.json"]}))+    except (ValueError, OSError, KeyError, TypeError, zipfile.BadZipFile) as e:+        print(json.dumps({"ok": False, "error": str(e)}), file=sys.stderr)+        return 1+    return 0+++if __name__ == "__main__":+    sys.exit(main())+--- a/SKILL.md+++ b/SKILL.md@@ -1,473 +1,490 @@⋯ 44 unchanged lines ⋯ | yes | STEP (`.step`) | exported ON the design machine so all 3D models (machine pins, contacts, MP markers) are baked in | | yes | Board/layout (`.kicad_pcb` or `.brd`) | what footprint+symbol generation parses | | yes | Schematic — `.kicad_sch` (EVERY sheet: root + sub-sheets) or `.sch` | source completeness + APM's hierarchical schematic view renders from these |+| EDA molecule | `<slug>-fab.zip` | both sides' copper, mask and silk, outline, Gerber job when available, and Excellon drill files |+| EDA molecule | `export.json` | `adom.export/1`, producer versions and hashes of sources and every zip member | | board shape | `.kicad_pro` + any `.kicad_sym`/`.pretty` used | APM render sources | | generated | GLB | OCCT conversion output — never user-supplied, always published | | generated | Metadata | OCCT footprint/symbol JSONs + the authored `package.json` | | board shape | Bundled 3D — `3d/<basename>` for every CUSTOM `(model ...)` | see below (#16); recipients render blank without it | | optional | Silk PNGs (top/bottom) | white artwork on transparent |-| optional | Extras | `.f3d`/`.usdz`/gerbers/BOM/CPL — carry what the EDA produced |+| optional | Extras | `.f3d`/`.usdz`/BOM/CPL — carry what the EDA produced | +**Fabrication handoff for EDA molecules.** Read [docs/fabrication.md](docs/fabrication.md).+Before conversion and again after final packaging, run+`python3 scripts/fab_bundle.py check <bundle-dir> --slug <slug>` from this+package's directory. A missing or stale fabrication bundle stops this publish;+return to the exporter for the missing files. Keep all existing GLB and metadata+checks until the registry rollout explicitly replaces that publishing path.+ **Bundle custom 3D models so the board renders on ANY machine (#16).** The whole-board GLB drives the wiki 3D viewer, so a board looks fine on the page even when its per-component models are NOT bundled — the gap only appears when⋯ 60 unchanged lines ⋯   <slug>.glb            # OCCT artifact — MUST be named exactly <slug>.glb (see below)   <name>.kicad_pcb|.brd # board source   <name>.kicad_sch|.sch # schematic source+  <name>.step          # native EDA source, with models resolved on the design machine+  <slug>-fab.zip       # required fabrication sources for EDA molecules+  export.json         # required versioned export record for EDA molecules   <name>_footprint.json # OCCT artifact — per-pad wire-attach Z   <name>_symbol.json    # OCCT artifact   README.md-  [extras: silk PNGs, .f3d, .usdz, gerbers, bom, cpl]+  [extras: silk PNGs, .f3d, .usdz, bom, cpl] ```  **Key components by MPN, never by value (#142).** The `slug` is the⋯ 144 unchanged lines ⋯ ```  Verify before declaring done:+- For an EDA molecule, the explicit push list includes `<slug>-fab.zip`,+  `export.json` and every recorded source. Download that complete set at the+  returned commit into a fresh directory and run `fab_bundle.py check` there;+  all hashes must match. See [docs/fabrication.md](docs/fabrication.md). - `adom-wiki api /api/v1/pages/<slug>` → `metadata.model_3d_path` is set - **the GLB at `model_3d_path` is byte-identical to YOUR gated conversion   output** — this is the anchoring proof. (Do NOT inspect the GLB for a⋯ 193 unchanged lines ⋯ - `kicad-export-molecule` / `fusion-export-molecule` / `altium-export-molecule` — the EDA → bundle front-ends that hand off here - `adom-step2glb` — the conversion CLI/service this skill drives - `molecule-pipeline` — the top-level router across the whole chain+--- a/tests/test_fab_bundle.py+++ b/tests/test_fab_bundle.py@@ -0,0 +1,86 @@+"""Regression checks for the shared export gate; run with unittest."""+import importlib.util+import json+import tempfile+import unittest+import zipfile+from pathlib import Path+from types import SimpleNamespace++SCRIPT = Path(__file__).resolve().parents[1] / "scripts/fab_bundle.py"+spec = importlib.util.spec_from_file_location("fab_bundle", SCRIPT)+fab = importlib.util.module_from_spec(spec)+spec.loader.exec_module(fab)+++class ExportTests(unittest.TestCase):+    def setUp(self):+        self.tmp = tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parent)+        self.addCleanup(self.tmp.cleanup)+        self.root = Path(self.tmp.name)+        assert self.root.resolve().is_relative_to(Path(__file__).resolve().parent)+        self.layers = self.root / "fab"+        self.layers.mkdir()+        for ext in ("gtl", "gbl", "gts", "gbs", "gto", "gbo", "gm1"):+            (self.layers / f"board.{ext}").write_bytes(b"G04 Test*\nX0Y0D02*\nX10Y10D01*\nM02*\n")+        (self.layers / "board-PTH.drl").write_bytes(b"M48\nMETRIC,TZ\nT01C0.3\n%\nT01\nX0Y0\nM30\n")+        (self.root / "board.step").write_bytes(b"STEP source")+        (self.root / "board.kicad_pcb").write_bytes(b"saved board")+        self.args = SimpleNamespace(root=str(self.root), slug="test-board", fab=str(self.layers),+                                    source=["step=board.step", "board_kicad_pcb=board.kicad_pcb"],+                                    tool="KiCad", tool_version="10.0.3", exporter="test",+                                    exporter_version="1", stackup=None)++    def test_preserves_original_members_and_checks_roundtrip(self):+        rec = fab.create(self.args)+        zipped = fab.read_zip(self.root / "test-board-fab.zip")+        self.assertEqual(zipped, {p.name: p.read_bytes() for p in self.layers.iterdir()})+        self.assertEqual(fab.check(self.root, "test-board"), rec)++    def test_board_change_invalidates_record(self):+        fab.create(self.args)+        (self.root / "board.kicad_pcb").write_bytes(b"new revision")+        with self.assertRaisesRegex(ValueError, "source changed"):+            fab.check(self.root, "test-board")++    def test_missing_bottom_layer_refused(self):+        (self.layers / "board.gbs").unlink()+        with self.assertRaisesRegex(ValueError, "mask_bottom"):+            fab.create(self.args)++    def test_empty_altium_outline_refused(self):+        (self.layers / "board.gm1").write_bytes(b"G04 Header*\nM02*\n")+        with self.assertRaisesRegex(ValueError, "no drawn segments"):+            fab.create(self.args)++    def test_stale_zip_record_refused_even_when_zip_hash_is_updated(self):+        rec = fab.create(self.args)+        zpath = self.root / "test-board-fab.zip"+        files = fab.read_zip(zpath)+        files["board.gtl"] += b"G04 altered*\n"+        with zipfile.ZipFile(zpath, "w") as z:+            for n, b in files.items():+                z.writestr(n, b)+        saved = rec["files"][-1]+        saved.update(fab.fingerprint(zpath.name, "fab_zip", zpath.read_bytes()))+        (self.root / "export.json").write_text(json.dumps(rec))+        with self.assertRaisesRegex(ValueError, "zip entry changed"):+            fab.check(self.root, "test-board")++    def test_zip_path_traversal_refused_without_extraction(self):+        path = self.root / "input.zip"+        with zipfile.ZipFile(path, "w") as z:+            z.writestr("../escape.gtl", b"anything")+        with self.assertRaisesRegex(ValueError, "relative file path"):+            fab.read_zip(path)++    def test_x2_beats_extension_and_identifies_npth(self):+        self.assertEqual(fab.role_for("misnamed.gtl", b"%TF.FileFunction,Legend,Bot*%"), "silk_bottom")+        self.assertEqual(fab.role_for("board.TXT", b"M48\n; #@! TF.FileFunction,NonPlated,1,2,NPTH\n"), "drill_unplated")+        self.assertEqual(fab.role_for("MpinTest-Plated.TXT", b"M48\nMETRIC,TZ\n"), "drill_plated")+        self.assertEqual(fab.role_for("inner.G2", b"G04 gerber*\n"), "copper_inner_2")+++if __name__ == "__main__":+    unittest.main()+