Adom Chip Fetcher

#!/bin/bash
# cf-pup-backfill.sh <MPN> <product-page-URL>
# "Browse as a real user" PDF backfill: navigates pup Chrome to the product page,
# auto-accepts the cookie banner (incl. shadow-DOM), scrapes all .pdf anchors,
# then pulls each via in-page fetch() with credentials so vendor anti-bot
# checks (Akamai/Cloudflare/etc.) pass with the real Chrome session.
#
# Every saved file is verified as %PDF magic before keeping. HTML denial pages
# never make it onto disk.

set -u
MPN="${1:?usage: cf-pup-backfill.sh <MPN> <URL>}"
URL="${2:?usage: cf-pup-backfill.sh <MPN> <URL>}"
LIB_DIR="/home/adom/project/chip-fetcher/library/$MPN"
mkdir -p "$LIB_DIR"

echo "=== pup-backfill: $MPN ==="
echo "  page: $URL"

adom-desktop browser_navigate "{\"sessionId\":\"chip-fetcher\",\"url\":\"$URL\"}" >/dev/null 2>&1
sleep 6

# Auto-accept cookies (DOM + shadow DOM)
adom-desktop browser_eval '{"sessionId":"chip-fetcher","expr":"(()=>{const tries=(root)=>{const btns=Array.from(root.querySelectorAll(\"button, a\"));const re=/^(accept all|accept all cookies|accept|i agree|allow all|ok)$/i;const m=btns.find(b=>re.test((b.textContent||\"\").trim()));if(m){m.click();return true;}return false;};if(tries(document))return\"accepted\";const seen=new Set();const walk=(root)=>{for(const el of root.querySelectorAll(\"*\")){if(el.shadowRoot&&!seen.has(el)){seen.add(el);if(tries(el.shadowRoot))return true;if(walk(el.shadowRoot))return true;}}return false;};return walk(document)?\"accepted-in-shadow\":\"no-banner\";})()"}' 2>/dev/null >/dev/null
sleep 2

# Verify not on a 404 / denial page
TITLE=$(adom-desktop browser_eval '{"sessionId":"chip-fetcher","expr":"document.title.slice(0,80)"}' 2>/dev/null \
  | python3 -c "import sys,json; print(json.load(sys.stdin).get('result',''))" 2>/dev/null)
if echo "$TITLE" | grep -qi "404\|access.*denied\|not found"; then
  echo "  SKIP: $TITLE"
  exit 0
fi

# Scrape PDF anchors
PDF_JSON=$(adom-desktop browser_eval '{"sessionId":"chip-fetcher","expr":"(()=>{const links=Array.from(document.querySelectorAll(\"a\")).filter(a=>/\\.pdf(\\?|$|#)/i.test(a.href||\"\"));const seen=new Set();const out=[];for(const a of links){const h=a.href;if(seen.has(h))continue;seen.add(h);const rowText=(a.closest(\"tr,li,article,section,div\")?.textContent||a.textContent||\"\").trim().replace(/\\s+/g,\" \").slice(0,80);out.push(h+\"|\"+rowText);}return JSON.stringify(out);})()"}' 2>/dev/null \
  | python3 -c "import sys,json; print(json.load(sys.stdin).get('result',''))" 2>/dev/null)

if [ -z "$PDF_JSON" ] || [ "$PDF_JSON" = "[]" ]; then
  echo "  no PDF anchors visible on rendered page"
  exit 0
fi

echo "$PDF_JSON" | python3 -c "
import sys, json
items = json.load(sys.stdin)
for entry in items:
    parts = entry.split('|', 1)
    if len(parts) == 2: print(parts[0] + '\t' + parts[1])
    else: print(parts[0] + '\t')
" > /tmp/cf-pup-pdfs.txt

DOC_COUNT=$(wc -l < /tmp/cf-pup-pdfs.txt)
echo "  found $DOC_COUNT PDF links"

COUNT=0
SKIPPED=0
FAILED=0
while IFS=$'\t' read -r PDF_URL TITLE; do
  [ -z "$PDF_URL" ] && continue
  if [ -n "$TITLE" ]; then
    SLUG=$(echo "$TITLE" | sed -E 's/[^a-zA-Z0-9-]+/-/g; s/^-+|-+$//g; s/--+/-/g' | cut -c1-40 | tr 'A-Z' 'a-z')
  fi
  if [ -z "${SLUG:-}" ]; then
    SLUG=$(basename "$PDF_URL" .pdf | tr 'A-Z' 'a-z' | cut -c1-40)
  fi
  OUT="$LIB_DIR/$MPN-$SLUG.pdf"
  if [ -f "$OUT" ]; then SKIPPED=$((SKIPPED + 1)); continue; fi

  # In-page fetch with credentials. Returns OK_<base64> or FAIL_<status>.
  EXPR=$(python3 -c "
import json
url = '$PDF_URL'
print(json.dumps(f'(async()=>{{try{{const r=await fetch({json.dumps(url)},{{credentials:\"include\"}});if(!r.ok)return\"FAIL_\"+r.status;const ab=await r.arrayBuffer();const b=new Uint8Array(ab);let s=\"\";for(let i=0;i<b.length;i++)s+=String.fromCharCode(b[i]);return\"OK_\"+btoa(s);}}catch(e){{return\"ERR_\"+e.message;}}}})()'))
")
  RESULT=$(adom-desktop browser_eval "{\"sessionId\":\"chip-fetcher\",\"expr\":$EXPR,\"awaitPromise\":true}" 2>/dev/null \
    | python3 -c "import sys,json; r=json.load(sys.stdin).get('result',''); print(r if isinstance(r,str) else '')" 2>/dev/null)

  if [[ "$RESULT" == OK_* ]]; then
    echo "${RESULT#OK_}" | base64 -d > "$OUT" 2>/dev/null
    SIZE=$(stat -c%s "$OUT" 2>/dev/null || echo 0)
    if [ "$SIZE" -gt 1000 ] && head -c 4 "$OUT" | grep -q "%PDF"; then
      echo "  + $MPN-$SLUG.pdf ($((SIZE/1024))KB)"
      COUNT=$((COUNT + 1))
    else
      rm -f "$OUT"; echo "  fail not-pdf: $SLUG"; FAILED=$((FAILED + 1))
    fi
  else
    echo "  fail $SLUG: $RESULT"; FAILED=$((FAILED + 1))
  fi
done < /tmp/cf-pup-pdfs.txt

echo "  pulled=$COUNT skipped=$SKIPPED failed=$FAILED"