Adom Chip Fetcher

#!/bin/bash
# cf-vendor-backfill.sh <MPN> <product-page-URL>
# Generic per-vendor PDF backfill: navigates pup to a product page, scrapes all
# anchor links pointing at .pdf, and curls each into library/<MPN>/.
#
# Reuses the active chip-fetcher tab (browser_navigate, not browser_open_tab).

set -u
MPN="${1:?usage: cf-vendor-backfill.sh <MPN> <URL>}"
URL="${2:?usage: cf-vendor-backfill.sh <MPN> <URL>}"

LIB_DIR="/home/adom/project/chip-fetcher/library/$MPN"
mkdir -p "$LIB_DIR"

echo "=== vendor backfill: $MPN ==="
echo "  product page: $URL"

adom-desktop browser_navigate "{\"sessionId\":\"chip-fetcher\",\"url\":\"$URL\"}" >/dev/null 2>&1
sleep 6

# Verify not 404
TITLE=$(adom-desktop browser_eval '{"sessionId":"chip-fetcher","expr":"document.title.slice(0,80)"}' 2>/dev/null \
  | python3 -c "import sys,json; print(json.load(sys.stdin).get('result',''))" 2>/dev/null)
if echo "$TITLE" | grep -qi "404\|not found\|page not"; then
  echo "  SKIP: 404 ($TITLE)"
  exit 0
fi

# Scrape all PDF anchor hrefs
PDF_JSON=$(adom-desktop browser_eval '{"sessionId":"chip-fetcher","expr":"(()=>{const links=Array.from(document.querySelectorAll(\"a\")).filter(a=>/\\.pdf(\\?|$|#)/i.test(a.href||\"\"));const seen=new Set();const out=[];for(const a of links){const h=a.href;if(seen.has(h))continue;seen.add(h);const rowText=(a.closest(\"tr,li,article,section,div\")?.textContent||a.textContent||\"\").trim().replace(/\\s+/g,\" \").slice(0,80);out.push(h+\"|\"+rowText);}return JSON.stringify(out);})()"}' 2>/dev/null \
  | python3 -c "import sys,json; print(json.load(sys.stdin).get('result',''))" 2>/dev/null)

if [ -z "$PDF_JSON" ] || [ "$PDF_JSON" = "[]" ]; then
  echo "  SKIP: no PDF links on page"
  exit 0
fi

echo "$PDF_JSON" | python3 -c "
import sys, json
items = json.load(sys.stdin)
for entry in items:
    parts = entry.split('|', 1)
    if len(parts) == 2:
        print(parts[0] + '\t' + parts[1])
    else:
        print(parts[0] + '\t')
" > /tmp/cf-vendor-pdfs.txt

DOC_COUNT=$(wc -l < /tmp/cf-vendor-pdfs.txt)
echo "  found $DOC_COUNT unique PDF links"

COUNT=0
SKIPPED=0
FAILED=0
while IFS=$'\t' read -r PDF_URL TITLE; do
  [ -z "$PDF_URL" ] && continue

  # Build slug from title; fall back to filename
  if [ -n "$TITLE" ]; then
    SLUG=$(echo "$TITLE" | sed -E 's/[^a-zA-Z0-9-]+/-/g; s/^-+|-+$//g; s/--+/-/g' | cut -c1-40 | tr 'A-Z' 'a-z')
  fi
  if [ -z "${SLUG:-}" ]; then
    SLUG=$(basename "$PDF_URL" .pdf | tr 'A-Z' 'a-z' | cut -c1-40)
  fi
  OUT="$LIB_DIR/$MPN-$SLUG.pdf"
  if [ -f "$OUT" ]; then
    SKIPPED=$((SKIPPED + 1)); continue
  fi

  HEAD=$(curl -sLI -A "Mozilla/5.0" "$PDF_URL" 2>/dev/null)
  CT=$(echo "$HEAD" | grep -i "^content-type:" | tail -1 | tr -d '\r' | awk '{print $2}')
  CL=$(echo "$HEAD" | grep -i "^content-length:" | tail -1 | tr -d '\r' | awk '{print $2}')

  if echo "$CT" | grep -qi "pdf"; then
    if [ -n "$CL" ] && [ "$CL" -gt 26214400 ]; then
      echo "  skip large: $SLUG ($((CL/1024/1024))MB)"
      SKIPPED=$((SKIPPED + 1)); continue
    fi
    if curl -sL -A "Mozilla/5.0" -o "$OUT" "$PDF_URL" 2>/dev/null; then
      SIZE=$(stat -c%s "$OUT" 2>/dev/null || echo 0)
      if [ "$SIZE" -gt 1000 ]; then
        echo "  + $MPN-$SLUG.pdf ($((SIZE/1024))KB)"
        COUNT=$((COUNT + 1))
      else
        rm -f "$OUT"; FAILED=$((FAILED + 1))
      fi
    else
      FAILED=$((FAILED + 1))
    fi
  else
    SKIPPED=$((SKIPPED + 1))
  fi
done < /tmp/cf-vendor-pdfs.txt

echo "  pulled=$COUNT skipped=$SKIPPED failed=$FAILED"