Adom Chip Fetcher

#!/bin/bash
# cf-ti-backfill.sh <MPN>
# Open the TI product page in pup, scrape the "Technical documentation" section
# for /lit/pdf/<id> URLs, curl each into library/<MPN>/<MPN>-<id>.pdf.
# Skips files >25MB and dupes already on disk.

set -u
MPN="${1:?usage: cf-ti-backfill.sh <MPN>}"
LIB_DIR="/home/adom/project/chip-fetcher/library/$MPN"
mkdir -p "$LIB_DIR"

echo "=== TI backfill: $MPN ==="

# TI product pages use the BASE part number (no package suffix).
# Progressive truncation: try as-is, then strip 1 char at a time until 200 or 4 chars left.
BASE=""
TRY="$MPN"
while [ ${#TRY} -ge 4 ]; do
  CODE=$(curl -sL -A "Mozilla/5.0" -o /dev/null -w "%{http_code}" "https://www.ti.com/product/$TRY")
  if [ "$CODE" = "200" ]; then
    BASE="$TRY"
    break
  fi
  TRY="${TRY%?}"
done
if [ -z "$BASE" ]; then
  echo "  SKIP: no TI page found for any prefix of $MPN"
  exit 0
fi
echo "  base part: $BASE"

# Open product page — reuse the active tab (don't pile up tabs).
adom-desktop browser_navigate "{\"sessionId\":\"chip-fetcher\",\"url\":\"https://www.ti.com/product/$BASE\"}" >/dev/null 2>&1
sleep 5

# Verify we landed on a real product page (not 404)
TITLE=$(adom-desktop browser_eval '{"sessionId":"chip-fetcher","expr":"document.title.slice(0,80)"}' 2>/dev/null \
  | python3 -c "import sys,json; print(json.load(sys.stdin).get('result',''))" 2>/dev/null)
if echo "$TITLE" | grep -qi "404\|not found\|error"; then
  echo "  SKIP: TI 404 for $MPN ($TITLE)"
  exit 0
fi

# Scrape /lit/pdf/<id> URLs from the Technical Documentation section
LIT_JSON=$(adom-desktop browser_eval '{"sessionId":"chip-fetcher","expr":"(()=>{const hRe=new RegExp(\"technical documentation\",\"i\"); const heads=Array.from(document.querySelectorAll(\"h1,h2,h3,h4\")); const hd=heads.find(h=>hRe.test(h.textContent||\"\")); if(!hd) return \"NO\"; const sec=hd.closest(\"section,div\"); const re=new RegExp(\"^https://www.ti.com/lit/pdf/[a-zA-Z0-9_-]+$\"); const links=Array.from(sec.querySelectorAll(\"a\")).filter(a=>re.test(a.href)); const seen=new Set(); const out=[]; for(const a of links){const id=a.href.split(\"/\").pop(); if(seen.has(id)) continue; seen.add(id); const rowText=(a.closest(\"tr,li,article,div\")?.textContent||a.textContent||\"\").trim().replace(/\\s+/g,\" \").slice(0,80); out.push(id+\"|\"+rowText);} return JSON.stringify(out);})()"}' 2>/dev/null \
  | python3 -c "import sys,json; print(json.load(sys.stdin).get('result',''))" 2>/dev/null)

if [ -z "$LIT_JSON" ] || [ "$LIT_JSON" = "NO" ]; then
  echo "  SKIP: no Technical Documentation section found"
  exit 0
fi

# Parse JSON list (each entry: "<docid>|<title>")
echo "$LIT_JSON" | python3 -c "
import sys, json
items = json.load(sys.stdin)
for entry in items:
    parts = entry.split('|', 1)
    if len(parts) == 2:
        print(parts[0] + '\t' + parts[1])
    else:
        print(parts[0] + '\t')
" > /tmp/cf-ti-doc-list.txt

DOC_COUNT=$(wc -l < /tmp/cf-ti-doc-list.txt)
echo "  found $DOC_COUNT unique docs"

# Curl each into library/<MPN>/
COUNT=0
SKIPPED=0
FAILED=0
while IFS=$'\t' read -r DOCID TITLE; do
  [ -z "$DOCID" ] && continue
  # Sanitize title slug for filename
  SLUG=$(echo "$TITLE" | sed -E 's/[^a-zA-Z0-9-]+/-/g; s/^-+|-+$//g; s/--+/-/g' | cut -c1-40 | tr 'A-Z' 'a-z')
  [ -z "$SLUG" ] && SLUG="$DOCID"
  OUT="$LIB_DIR/$MPN-$SLUG.pdf"

  # Skip if already exists
  if [ -f "$OUT" ]; then
    SKIPPED=$((SKIPPED + 1))
    continue
  fi

  URL="https://www.ti.com/lit/pdf/$DOCID"
  # HEAD first to check size + type
  HEAD=$(curl -sLI -A "Mozilla/5.0" "$URL" 2>/dev/null)
  CT=$(echo "$HEAD" | grep -i "^content-type:" | tail -1 | tr -d '\r' | awk '{print $2}')
  CL=$(echo "$HEAD" | grep -i "^content-length:" | tail -1 | tr -d '\r' | awk '{print $2}')

  if echo "$CT" | grep -q "pdf"; then
    if [ -n "$CL" ] && [ "$CL" -gt 26214400 ]; then
      echo "  skip $DOCID ($((CL/1024/1024))MB > 25MB cap)"
      SKIPPED=$((SKIPPED + 1))
      continue
    fi
    if curl -sL -A "Mozilla/5.0" -o "$OUT" "$URL" 2>/dev/null; then
      SIZE=$(stat -c%s "$OUT" 2>/dev/null || echo 0)
      if [ "$SIZE" -gt 1000 ]; then
        echo "  + $MPN-$SLUG.pdf ($((SIZE/1024))KB)"
        COUNT=$((COUNT + 1))
      else
        rm -f "$OUT"
        FAILED=$((FAILED + 1))
      fi
    else
      FAILED=$((FAILED + 1))
    fi
  else
    SKIPPED=$((SKIPPED + 1))
  fi
done < /tmp/cf-ti-doc-list.txt

echo "  pulled=$COUNT skipped=$SKIPPED failed=$FAILED"