Service Containers

// Reads the watchdog's own on-disk state. This app NEVER probes a service
// itself: service-watchdog.sh already does that every 5 minutes and writes
// its verdicts to $WD_STATE_DIR. Probing again here would double the load
// and, for Mouser and DigiKey, spend billable vendor quota just to draw a
// dot green. Read the verdict, don't re-derive it.

import { readFileSync, readdirSync, statSync, existsSync } from 'fs';
import { join } from 'path';

export const STATE_DIR = process.env.WD_STATE_DIR || '/home/adom/.watchdog-state';
const BIN = '/home/adom/bin';

const WATCHDOG_LOGS = [
  '/tmp/service-watchdog.log',
  '/tmp/kicad-watchdog.log',
  '/tmp/parts-vendor-watchdog.log',
];

// Human copy for each check key. The dashboard says what a failure MEANS,
// not just which probe tripped , "auto-deploy is dead" beats "git-auth down".
const CHECK_META = {
  'health':            { name: 'Health',      blurb: 'Public /health answers 200' },
  'deep':              { name: 'Deep probe',  blurb: 'A real API call, not just the port' },
  'cron':              { name: 'Cron',        blurb: 'Scheduled jobs can still run on the container' },
  'git-auth':          { name: 'Git auth',    blurb: 'Auto-deploy from main still works' },
  '5xx-rate':          { name: '5xx rate',    blurb: 'Recent server errors in the log' },
  '5xx':               { name: '5xx rate',    blurb: 'Recent server errors in the log' },
  'quota':             { name: 'Vendor quota',blurb: 'Daily API allowance not exhausted' },
  'freshness':         { name: 'Data age',    blurb: 'Backing dataset is being refreshed' },
  'credential-health': { name: 'Credentials', blurb: 'Stored credentials still valid' },
  'deploy':            { name: 'Deploy',      blurb: 'Latest main is actually running' },
  'git':               { name: 'Git auth',    blurb: 'Auto-deploy from main still works' },
};

// Service identity. Parsed from the watchdog TARGETS rows where possible so
// this can't drift; the map below only supplies display names and the notes
// that aren't encoded in the watchdog config.
const DISPLAY = {
  'mouser':            { name: 'Mouser',        repo: 'john/service-mouser',   note: 'Parts search vendor' },
  'digikey':           { name: 'DigiKey',       repo: 'john/service-digikey',  note: 'Parts search vendor' },
  'jlcpcb':            { name: 'JLCPCB',        repo: 'john/service-jlcpcb',   note: 'Parts search vendor, local SQLite' },
  'tts':               { name: 'TTS',           repo: 'john/service-tts',      note: 'Speech for hands-free' },
  'step2glb':          { name: 'STEP to GLB',   repo: 'john/service-step2glb', note: 'Chip thumbnails and 3D' },
  'oauth':             { name: 'OAuth Gateway', repo: 'john/service-oauth',    note: 'Static callback relay' },
  'kicad-v10-default': { name: 'KiCad CLI',     repo: 'john/service-kicad',    note: 'KiCad 10 headless' },
};

// Some watchdogs write under their own label but describe a facet of an
// existing service. Fold them in rather than inventing a second card.
const MERGE_INTO = { 'kicad-cred': 'kicad-v10-default' };

// Labels whose containers are deprecated. Their state files linger, but
// showing them as failing services would be a lie , they are off on purpose.
const RETIRED = {
  'wiki':          'Old wiki container, deprecated. wiki.adom.inc is separate and owned elsewhere.',
  'wiki-adom-inc': 'Monitored elsewhere, not ours to run.',
};

// Display names for retired labels, which have no DISPLAY entry.
const RETIRED_NAMES = {
  'wiki':          'Old wiki',
  'wiki-adom-inc': 'wiki.adom.inc',
};

function readTargets() {
  const out = {};
  try {
    const src = readFileSync(join(BIN, 'service-watchdog.sh'), 'utf8');
    const block = src.split('TARGETS=(')[1]?.split('\n)')[0] || '';
    for (const line of block.split('\n')) {
      const m = line.match(/^\s*"([^"]+)"/);
      if (!m) continue;
      const f = m[1].split('|');
      out[f[0]] = { ssh: f[1] || '', host: f[2] || '', port: f[3] || '', start: f[5] || '' };
    }
  } catch {}
  try {
    const src = readFileSync(join(BIN, 'kicad-watchdog-of-watchdogs.sh'), 'utf8');
    const block = src.split('TARGETS=(')[1]?.split('\n)')[0] || '';
    for (const line of block.split('\n')) {
      const m = line.match(/^\s*"([^"]+)"/);
      if (!m) continue;
      const f = m[1].split(':');
      out['kicad-' + f[0]] = { ssh: f[1] || '', host: f[2] || '', port: '8780', start: '' };
    }
  } catch {}
  return out;
}

// A state file holds either "ok" or "<kind>|<reason>|<since>".
function parseStateFile(path) {
  let raw = '';
  try { raw = readFileSync(path, 'utf8').trim(); } catch { return null; }
  if (!raw) return null;
  const [kind, reason = '', since = ''] = raw.split('|');
  let mtime = null;
  try { mtime = statSync(path).mtime.toISOString(); } catch {}
  return { kind, reason, since, mtime };
}

// Roll the per-check verdicts up to one service verdict.
//   down    - a confirmed fault, already announced
//   pending - failing but inside the debounce window, not yet announced
//   ok      - everything green
function rollup(checks) {
  if (checks.some(c => c.kind === 'down')) return 'down';
  if (checks.some(c => c.kind === 'pending')) return 'pending';
  return checks.length ? 'ok' : 'unknown';
}

export function readLogTail(limit = 60) {
  const lines = [];
  for (const p of WATCHDOG_LOGS) {
    if (!existsSync(p)) continue;
    let txt = '';
    try { txt = readFileSync(p, 'utf8'); } catch { continue; }
    const src = p.split('/').pop().replace('.log', '');
    for (const l of txt.trim().split('\n').slice(-limit)) {
      if (!l.trim()) continue;
      const m = l.match(/^\[([^\]]+)\]\s*(.*)$/);
      lines.push({ ts: m ? m[1] : '', text: m ? m[2] : l, src });
    }
  }
  // Newest first, and only entries that actually carry a timestamp sort well.
  lines.sort((a, b) => (b.ts || '').localeCompare(a.ts || ''));
  return lines.slice(0, limit);
}

export function readState() {
  const targets = readTargets();
  const byLabel = {};

  let files = [];
  try { files = readdirSync(STATE_DIR); } catch {}

  for (const f of files) {
    if (!f.endsWith('.state')) continue;
    const stem = f.slice(0, -'.state'.length);
    // Split on the LAST hyphen group that names a known check, so labels
    // containing hyphens (kicad-v10-default) survive intact.
    let label = stem, check = '';
    const known = Object.keys(CHECK_META).sort((a, b) => b.length - a.length);
    for (const k of known) {
      if (stem.endsWith('-' + k)) { check = k; label = stem.slice(0, -(k.length + 1)); break; }
    }
    if (!check) continue;

    const st = parseStateFile(join(STATE_DIR, f));
    if (!st) continue;

    if (MERGE_INTO[label]) { check = 'cred-' + check; label = MERGE_INTO[label]; }

    byLabel[label] ||= { label, checks: [] };
    const base = check.startsWith('cred-') ? check.slice(5) : check;
    const meta = check.startsWith('cred-')
      ? { name: 'Credential ' + (CHECK_META[base]?.name || base).toLowerCase(),
          blurb: 'Stored credentials still valid' }
      : (CHECK_META[check] || { name: check, blurb: '' });
    byLabel[label].checks.push({
      key: check, name: meta.name, blurb: meta.blurb,
      kind: st.kind, reason: st.reason, since: st.since, checkedAt: st.mtime,
    });
  }

  const services = [];
  const retired = [];

  for (const label of Object.keys(byLabel)) {
    const entry = byLabel[label];
    const t = targets[label] || {};
    const d = DISPLAY[label] || {};
    entry.checks.sort((a, b) => a.name.localeCompare(b.name));

    const row = {
      label,
      name: d.name || label,
      repo: d.repo || '',
      note: d.note || '',
      host: t.host || '',
      port: t.port || '',
      ssh: t.ssh || '',
      startCmd: t.start || '',
      checks: entry.checks,
      status: rollup(entry.checks),
      lastCheckedAt: entry.checks.map(c => c.checkedAt).filter(Boolean).sort().pop() || null,
    };

    if (RETIRED[label]) {
      retired.push({ ...row, name: RETIRED_NAMES[label] || row.name, why: RETIRED[label] });
      continue;
    }
    services.push(row);
  }

  // Faults first, then alphabetical, so what needs attention is at the top.
  const rank = { down: 0, pending: 1, unknown: 2, ok: 3 };
  services.sort((a, b) => (rank[a.status] - rank[b.status]) || a.name.localeCompare(b.name));

  const counts = { total: services.length, ok: 0, pending: 0, down: 0, unknown: 0 };
  for (const s of services) counts[s.status]++;

  // How long since the watchdog last wrote ANY verdict. If this grows past
  // ~10 minutes the watchdog itself has stopped, which is the failure mode
  // that hid the 7.3h OAuth outage , so the dashboard has to show it.
  let sweepAgeS = null;
  const newest = services.map(s => s.lastCheckedAt).filter(Boolean).sort().pop();
  if (newest) sweepAgeS = Math.max(0, Math.round((Date.now() - new Date(newest).getTime()) / 1000));

  return {
    generatedAt: new Date().toISOString(),
    counts,
    sweepAgeS,
    sweepStale: sweepAgeS != null && sweepAgeS > 900,
    services,
    retired,
    log: readLogTail(40),
  };
}