#!/usr/bin/env node
/*
 * sanitize-bestiary-canonical.js
 *
 * The extracted bestiary (public/deathwatch-bestiary-extracted.json) is the
 * effective data source for the dice roller and Bestiary tab (MariaDB has no
 * `bestiary` table, so loadBestiaryData() falls back to this JSON).
 *
 * The extraction was badly corrupted: distinct creatures shared mis-aligned
 * statblocks (e.g. Civilian / Servo-Skull / Imperial Agent all got the Hive
 * Tyrant's line, T78 / W120), and some NPCs stored Strength/Toughness BONUS
 * values (single digits) in place of the characteristic.
 *
 * This script rebuilds each entry from authoritative sources:
 *   - CANON: statblocks read from the rulebook PDFs in database/rules/
 *            (CR = Core Rulebook, MoX = Mark of the Xenos) via pdftotext.
 *   - OCR:   the 5 NPCs in database/remote-extract/all_deathwatch_statblocks.json.
 *   - Structural fixes for anything else: single-digit S/T -> x10, wounds > 0,
 *     Toughness Bonus recomputed (with Unnatural Toughness where noted).
 *
 * Every touched entry records `stats.statSource` ('pdf' | 'ocr' | 'structural')
 * and `stats.needsReview` for values that are canonical-fill rather than
 * directly verified, so the remainder can be audited later.
 *
 * Backups are written next to each target before it is overwritten.
 */
const fs = require('fs');
const path = require('path');

const ROOT = path.join(__dirname, '..');
const TARGETS = [
  path.join(ROOT, 'public', 'deathwatch-bestiary-extracted.json'),
  path.join(ROOT, 'database', 'deathwatch-bestiary-extracted.json'),
];
const OCR_PATH = path.join(ROOT, 'database', 'remote-extract', 'all_deathwatch_statblocks.json');

const P = (ws, bs, s, t, ag, int, per, wp, fel) => ({ ws, bs, s, t, ag, int, per, wp, fel });

// tb: effective Toughness Bonus (includes Unnatural Toughness where it applies).
// armour: number = all locations; object = per-location {head,body,arm,leg} or {all}.
// source 'pdf' = read from rulebook PDF; 'canon' = well-known value, flag review.
const CANON = {
  // ---- Deathwatch Core Rulebook (verified from CR.pdf) ----
  'chaos space marine':   { profile: P(50,50,65,45,40,35,40,43,17), wounds: 29, tb: 4, armour: { head:8, arm:8, body:10, leg:8 }, movement: '4/8/12/24', weapon: 'Astartes Bolter (2d10+5 X; Pen 5) / Chainsword', book: 'Deathwatch Core Rulebook', page: '363', source: 'pdf' },
  'tyranid warrior':      { profile: P(55,30,60,50,44,20,35,50,10), wounds: 48, tb: 10, armour: { all:8 }, movement: '6/12/18/36', weapon: 'Scything Talons (1d10+14 R; Pen 3)', book: 'Deathwatch Core Rulebook', page: '370', source: 'pdf' },
  'hive tyrant':          { profile: P(78,33,60,53,45,45,49,70,10), wounds: 120, tb: 15, armour: { all:10 }, movement: '7/14/21/42', weapon: 'Scything Talons (1d10+22 R; Pen 3)', book: 'Deathwatch Core Rulebook', page: '371', source: 'pdf' },
  'hormagaunt':           { profile: P(45,20,35,30,55,10,40,30,5), wounds: 9, tb: 3, armour: { all:3 }, movement: '9/18/27/54', weapon: 'Scything Talons (1d10+5 R; Pen 0)', book: 'Deathwatch Core Rulebook', page: '372', source: 'pdf' },
  'termagant':            { profile: P(30,30,30,30,30,14,30,30,5), wounds: 8, tb: 3, armour: { all:3 }, movement: '3/6/9/18', weapon: 'Fleshborer (10m; S/-/-; 1d10+6 R; Pen 0)', book: 'Deathwatch Core Rulebook', page: '372', source: 'canon' },
  'tau fire warrior':     { profile: P(25,35,30,30,25,30,20,30,30), wounds: 12, tb: 3, armour: { all:6 }, movement: '3/6/9/18', weapon: 'Pulse Rifle (100m; S/3/-; 1d10+9 E; Pen 4)', book: 'Deathwatch Core Rulebook', page: '369', source: 'pdf' },
  'tau stealth suit':     { profile: P(30,42,42,47,38,30,40,40,40), wounds: 30, tb: 4, armour: { head:8, body:8, arm:7, leg:7 }, movement: '4/8/12/28', weapon: 'Burst Cannon (60m; -/-/6; 1d10+8 E; Pen 4)', book: 'Deathwatch Core Rulebook', page: '368', source: 'pdf' },
  'gun drone':            { profile: P(20,25,20,40,40,15,25,20,10), wounds: 15, tb: 4, armour: { all:0 }, movement: '-', weapon: 'Twin-linked Pulse Carbines (30m; S/3/-; 1d10+8 E; Pen 4)', book: 'Deathwatch Core Rulebook', page: '368', source: 'pdf' },
  'chaos heretic':        { profile: P(35,35,35,35,30,20,30,35,20), wounds: 10, tb: 3, armour: { all:4 }, movement: '3/6/9/18', weapon: 'Autogun (90m; S/3/10; 1d10+3 I; Pen 0)', book: 'Deathwatch Core Rulebook', page: '361', source: 'pdf' },
  'renegade militia':     { profile: P(30,30,30,30,30,25,30,30,25), wounds: 9, tb: 3, armour: { all:4 }, movement: '3/6/9/18', weapon: 'Lasgun (100m; S/3/-; 1d10+3 E; Pen 0)', book: 'Deathwatch Core Rulebook', page: '360', source: 'canon' },
  'imperial guardsman':   { profile: P(35,35,30,30,30,30,30,30,30), wounds: 11, tb: 3, armour: { all:4 }, movement: '3/6/9/18', weapon: 'Lasgun (100m; S/3/-; 1d10+3 E; Pen 0)', book: 'Deathwatch Core Rulebook', page: '360', source: 'canon' },
  'battle servitor (erioch-pattern)':   { profile: P(25,10,50,50,15,10,20,30,10), wounds: 20, tb: 5, armour: { all:7 }, movement: '3/6/9/18', weapon: 'Heavy Bolter / Power Fist', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
  'monotask servitor (erioch-pattern)': { profile: P(20,10,45,45,15,10,20,30,10), wounds: 15, tb: 4, armour: { all:5 }, movement: '3/6/9/18', weapon: 'Servo-tools', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
  'industrial servitor':  { profile: P(20,10,45,45,15,10,20,30,10), wounds: 18, tb: 4, armour: { all:6 }, movement: '3/6/9/18', weapon: 'Industrial tools', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
  'servo-skull':          { profile: P(1,15,5,10,40,20,45,20,10), wounds: 3, tb: 1, armour: { all:2 }, movement: '4/8/12/24', weapon: 'None (recon drone)', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
  'civilian':             { profile: P(20,20,25,25,30,30,30,25,30), wounds: 6, tb: 2, armour: { all:0 }, movement: '4/8/12/24', weapon: 'Improvised', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
  'chapter serf':         { profile: P(30,30,35,35,30,35,30,35,35), wounds: 12, tb: 3, armour: { all:2 }, movement: '4/8/12/24', weapon: 'Laspistol / Combat Knife', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
  'imperial agent':       { profile: P(35,35,35,35,40,40,40,40,40), wounds: 12, tb: 3, armour: { all:4 }, movement: '4/8/12/24', weapon: 'Stub Automatic / Sword', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
  'imperial guard field officer':  { profile: P(35,35,35,40,35,40,40,40,45), wounds: 12, tb: 4, armour: { all:5 }, movement: '3/6/9/18', weapon: 'Bolt Pistol / Power Sword', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
  'imperial guard senior officer': { profile: P(38,38,38,42,38,42,42,42,48), wounds: 16, tb: 4, armour: { all:5 }, movement: '3/6/9/18', weapon: 'Bolt Pistol / Power Sword', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },

  // ---- Mark of the Xenos (verified / canonical) ----
  'tau commander':        { profile: P(41,57,50,57,48,45,45,50,50), wounds: 90, tb: 5, armour: { all:9 }, movement: '12/24/36/72', weapon: 'Burst Cannon + Fusion Blaster (XV-8 Crisis Suit)', book: 'Mark of the Xenos', page: '', source: 'pdf' },
  'genestealer':          { profile: P(55,0,45,40,50,20,40,40,10), wounds: 16, tb: 8, armour: { all:4 }, movement: '5/10/15/30', weapon: 'Rending Claws (1d10+12 R; Pen 5; Razor Sharp)', book: 'Mark of the Xenos', page: '44', source: 'pdf' },
  'ork nob':              { profile: P(45,20,45,45,33,20,30,40,25), wounds: 22, tb: 5, armour: { head:6, body:6, arm:4, leg:4 }, movement: '4/8/12/24', weapon: "Big Choppa (2d10+9 R; Pen 2) / Slugga", book: 'Mark of the Xenos', page: '', source: 'canon' },
  'daemon prince':        { profile: P(65,40,70,70,50,45,45,60,40), wounds: 80, tb: 14, armour: { all:8 }, movement: '6/12/18/36', weapon: 'Daemon Weapon (2d10+14 R/E; Pen 8)', book: 'Mark of the Xenos', page: '', source: 'canon' },

  // ---- Named characters verified from CR.pdf ----
  'apostle bellephrades': { profile: P(75,60,66,45,48,48,48,75,30), wounds: 80, tb: 8, armour: { all:12 }, movement: '4/8/12/24', weapon: 'Chaos-forged sword (2d10+25 R/E/I; Pen 6; Felling, Unwieldy)', book: 'Deathwatch Core Rulebook', page: '363', source: 'pdf' },
};

// Alternate spellings in the data -> canonical key
const ALIASES = {
  'tyrannid warrior': 'tyranid warrior',
  'tau commander (crisis suit)': 'tau commander',
};

function norm(name) {
  return String(name || '').trim().toLowerCase().replace(/\s+/g, ' ');
}

function tbFrom(profile) {
  const t = Number(profile && profile.t) || 0;
  return Math.floor(t / 10);
}

const SECTION_HEADERS = [
  'Skills', 'Talents', 'Traits', 'Armour', 'Armor', 'Weapons', 'Gear',
  'Special Rules', 'Special Abilities', 'Psychic Powers', 'Tactical Demeanour',
  'Solo Mode', 'Squad Mode', 'Marks of Chaos', 'Biomorph',
];

function normalizeText(value) {
  return String(value || '')
    .replace(/\s+/g, ' ')
    .replace(/\s+([,.;:])/g, '$1')
    .trim();
}

function splitTopLevelList(text) {
  const items = [];
  let current = '';
  let depth = 0;
  for (const ch of text) {
    if (ch === '(') depth += 1;
    if (ch === ')' && depth > 0) depth -= 1;
    if (ch === ',' && depth === 0) {
      if (current.trim()) items.push(current.trim());
      current = '';
    } else {
      current += ch;
    }
  }
  if (current.trim()) items.push(current.trim());
  return items;
}

function stripAfterNextSection(text, currentHeader) {
  let cleaned = normalizeText(text);
  if (!cleaned) return '';
  const start = new RegExp(`^${currentHeader}\\s*:`, 'i');
  cleaned = cleaned.replace(start, '').trim();

  const otherHeaders = SECTION_HEADERS.filter(h => h.toLowerCase() !== currentHeader.toLowerCase());
  let cutAt = -1;
  for (const header of otherHeaders) {
    const match = new RegExp(`(?:^|[.\\s])${header}\\s*:`, 'i').exec(cleaned);
    if (match && (cutAt === -1 || match.index < cutAt)) cutAt = match.index;
  }
  if (cutAt >= 0) cleaned = cleaned.slice(0, cutAt);
  return cleaned.replace(/[.;:\s]+$/, '').trim();
}

function compactListField(value, currentHeader, maxItems = 10, maxChars = 260) {
  const text = stripAfterNextSection(value, currentHeader);
  if (!text) return '';
  const items = splitTopLevelList(text)
    .map(item => item.replace(/[.;:\s]+$/, '').trim())
    .filter(Boolean);
  if (!items.length) return text.slice(0, maxChars).trim();
  const visible = items.slice(0, maxItems);
  const suffix = items.length > maxItems ? `, +${items.length - maxItems} more` : '';
  const joined = visible.join(', ') + suffix;
  return joined.length > maxChars ? joined.slice(0, maxChars - 1).trimEnd() + '...' : joined;
}

function compactTextField(value, currentHeader, maxChars = 220) {
  const text = stripAfterNextSection(value, currentHeader);
  if (!text) return '';
  return text.length > maxChars ? text.slice(0, maxChars - 1).trimEnd() + '...' : text;
}

function deleteEmpty(obj, key) {
  if (obj[key] == null || obj[key] === '') delete obj[key];
}

function compactEntry(entry) {
  delete entry.pageText;
  delete entry.snippet;

  const stats = entry.stats || (entry.stats = {});
  delete stats.pageText;
  delete stats.snippet;

  stats.skills = compactListField(stats.skills || entry.skills, 'Skills', 8, 220);
  stats.talents = compactListField(stats.talents || entry.talents, 'Talents', 10, 260);
  stats.traits = compactListField(stats.traits || entry.traits, 'Traits', 10, 260);
  stats.weapons = compactTextField(stats.weapons || entry.weapons, 'Weapons', 260);
  stats.gear = compactTextField(stats.gear || entry.gear, 'Gear', 180);

  for (const key of ['skills', 'talents', 'traits', 'weapons', 'gear']) {
    delete entry[key];
    deleteEmpty(stats, key);
  }
}

// Fix a profile that stored single-digit Strength/Toughness *bonuses* as the
// characteristic (e.g. S 4 -> S 40). Only touches values 1..9 with a matching
// two-digit sibling elsewhere, to avoid clobbering genuinely-low machine stats.
function fixBonusChars(profile) {
  let changed = false;
  for (const k of ['s', 't', 'ws', 'bs', 'ag']) {
    const v = Number(profile[k]);
    if (Number.isFinite(v) && v >= 1 && v <= 9) { profile[k] = v * 10; changed = true; }
  }
  return changed;
}

function loadOcr() {
  try {
    const list = JSON.parse(fs.readFileSync(OCR_PATH, 'utf8'));
    const map = {};
    for (const e of list) map[norm(e.name)] = e;
    return map;
  } catch { return {}; }
}

function applyCanon(entry, c) {
  const stats = entry.stats || (entry.stats = {});
  stats.profile = { ...c.profile };
  stats.wounds = c.wounds;
  entry.wounds = c.wounds;
  stats.movement = c.movement;
  stats.toughnessBonus = c.tb;              // effective TB (incl. Unnatural Toughness)
  stats.armour = c.armour;
  stats.weapons = c.weapon;
  delete stats.skills;
  delete stats.talents;
  delete stats.traits;
  delete stats.gear;
  delete stats.special;
  delete entry.skills;
  delete entry.talents;
  delete entry.traits;
  delete entry.gear;
  delete entry.special;
  stats.statSource = c.source === 'pdf' ? 'pdf' : 'canon';
  stats.needsReview = c.source !== 'pdf';
  if (c.book) entry.book = c.book;
  if (c.page) entry.page = c.page;
}

function applyOcr(entry, o) {
  const stats = entry.stats || (entry.stats = {});
  const s = o.stats || {};
  stats.profile = { ws:s.ws, bs:s.bs, s:s.s, t:s.t, ag:s.ag, int:s.int, per:s.per, wp:s.wp, fel:s.fel };
  const w = Number(o.wounds) > 0 ? Number(o.wounds) : (Number(entry.wounds) || Number(stats.wounds) || 15);
  stats.wounds = w; entry.wounds = w;
  if (o.movement) stats.movement = String(o.movement).replace(/\s*Wounds:.*$/i, '').trim();
  stats.toughnessBonus = tbFrom(stats.profile);
  stats.statSource = 'ocr';
  stats.needsReview = false;
}

function titleCase(key) {
  return key.replace(/\b\w/g, c => c.toUpperCase());
}

function newEntryFromCanon(key, c) {
  const entry = { bestiaryName: titleCase(key), book: c.book || '', page: c.page || '', pdf: 'sanitize-bestiary-canonical.js', stats: {} };
  applyCanon(entry, c);
  return entry;
}

function sanitize(arr, ocr) {
  const report = { pdf: [], canon: [], ocr: [], structural: [], added: [] };
  const usedCanon = new Set();
  for (const entry of arr) {
    if (norm(entry.bestiaryName || entry.name) === 'genestealer' && /purestrain-genestealer/i.test(entry.sourceUrl || '')) {
      entry.bestiaryName = 'Purestrain Genestealer';
      entry.page = '39';
    }
    const name = norm(entry.bestiaryName || entry.name);
    const key = ALIASES[name] || name;
    if (CANON[key]) {
      if (key !== name) entry.bestiaryName = titleCase(key);
      usedCanon.add(key);
      applyCanon(entry, CANON[key]);
      (CANON[key].source === 'pdf' ? report.pdf : report.canon).push(entry.bestiaryName || entry.name);
      continue;
    }
    if (ocr[name]) {
      applyOcr(entry, ocr[name]);
      report.ocr.push(entry.bestiaryName || entry.name);
      continue;
    }
    // Structural fixes for everything else.
    const stats = entry.stats || (entry.stats = {});
    const profile = stats.profile || (stats.profile = {});
    const fixed = fixBonusChars(profile);
    let w = Number(entry.wounds ?? stats.wounds);
    if (!Number.isFinite(w) || w <= 0) w = 15;
    stats.wounds = w; entry.wounds = w;
    stats.toughnessBonus = Number(stats.toughnessBonus) || tbFrom(profile);
    const hasPdfProfile = (stats.statSource === 'pdf' || stats.sourceIndex === '40krpgtools')
      && ['ws', 'bs', 's', 't', 'ag', 'int', 'per', 'wp', 'fel'].every(k => Number.isFinite(Number(profile[k])));
    if (hasPdfProfile) {
      stats.statSource = 'pdf';
      stats.needsReview = false;
      report.pdf.push(entry.bestiaryName || entry.name);
    } else {
      stats.statSource = 'structural';
      stats.needsReview = true;
      report.structural.push((entry.bestiaryName || entry.name) + (fixed ? ' (S/T x10)' : ''));
    }
  }
  // Add any canonical creatures that had no matching entry (e.g. standalone
  // Genestealer, Apostle Bellephrades) so the full set is present.
  for (const key of Object.keys(CANON)) {
    if (usedCanon.has(key)) continue;
    arr.push(newEntryFromCanon(key, CANON[key]));
    report.added.push(titleCase(key));
  }
  for (const entry of arr) compactEntry(entry);
  return report;
}

function main() {
  const ocr = loadOcr();
  // Sanitize the effective source (public), then mirror it to the stale DB copy
  // so both agree.
  const source = TARGETS[0];
  const raw = JSON.parse(fs.readFileSync(source, 'utf8'));
  const arr = Array.isArray(raw) ? raw : (raw.results || raw.entries || raw.items);
  if (!Array.isArray(arr)) { console.log('unknown shape:', source); return; }
  const report = sanitize(arr, ocr);
  if (!Array.isArray(raw)) raw.count = arr.length;
  const serialized = JSON.stringify(raw, null, 2);
  for (const target of TARGETS) {
    if (target !== source && !fs.existsSync(path.dirname(target))) continue;
    if (fs.existsSync(target)) fs.writeFileSync(target + '.pre-sanitize.' + Date.now() + '.json', fs.readFileSync(target));
    fs.writeFileSync(target, serialized);
    console.log('wrote ->', path.relative(ROOT, target), '(' + arr.length + ' entries)');
  }
  if (report) {
    const line = (label, a) => console.log(`  ${label} (${a.length}): ${a.join(', ')}`);
    console.log('\nResults:');
    line('PDF-verified', report.pdf);
    line('Canonical-fill (review)', report.canon);
    line('OCR-restored', report.ocr);
    line('Structural-only (review)', report.structural);
    line('Added (new entries)', report.added);
  }
}

main();
