dwroller/scripts/hydrate-bestiary-from-40krpgtools.js

#!/usr/bin/env node
/*
 * Hydrate staged 40k RPG Tools index rows from local PDFs into the compact live
 * bestiary JSON. This does not scrape statblocks from the web; it uses the web
 * index as a page map and extracts only compact combat fields from local PDFs.
 */
const fs = require('fs');
const path = require('path');
const { execFileSync } = require('child_process');

const ROOT = path.join(__dirname, '..');
const INDEX_PATH = path.join(ROOT, 'database', 'remote-extract', '40krpgtools-bestiary-index.json');
const PUBLIC_PATH = path.join(ROOT, 'public', 'deathwatch-bestiary-extracted.json');
const DB_PATH = path.join(ROOT, 'database', 'deathwatch-bestiary-extracted.json');

const BOOKS = {
  'Mark of the Xenos': { pdf: path.join(ROOT, 'database', 'rules', 'MoX.pdf'), pageOffset: 6 },
  'First Founding': { pdf: path.join(ROOT, 'database', 'rules', 'FF.pdf'), pageOffset: 8 },
  'The Emperor Protects': { pdf: path.join(ROOT, 'database', 'rules', 'HtC.pdf'), pageOffset: 8 },
  'Rites of Battle': { pdf: path.join(ROOT, 'database', 'rules', 'RoB.pdf'), pageOffset: 8 },
  'Deathwatch Core Rulebook': { pdf: path.join(ROOT, 'database', 'rules', 'CR.pdf'), pageOffset: 0 },
};

const PROFILE_KEYS = ['ws', 'bs', 's', 't', 'ag', 'int', 'per', 'wp', 'fel'];
const RANK_RE = '(Troops?|Elite|Master|Minion|Personal|Vehicle|Horde)';

function argValue(name, fallback = null) {
  const prefix = `--${name}=`;
  const found = process.argv.find(arg => arg.startsWith(prefix));
  return found ? found.slice(prefix.length) : fallback;
}

function hasFlag(name) {
  return process.argv.includes(`--${name}`);
}

function norm(value) {
  return String(value || '')
    .toLowerCase()
    .replace(/[’']/g, '')
    .replace(/[^a-z0-9]+/g, ' ')
    .trim()
    .replace(/\s+/g, ' ');
}

function titleCase(value) {
  return String(value || '').replace(/\b\w/g, c => c.toUpperCase());
}

function textForPage(book, page) {
  const cfg = BOOKS[book];
  const listedPage = Number(page);
  const physicalStart = Math.max(1, listedPage);
  const physicalEnd = listedPage + cfg.pageOffset + 2;
  return execFileSync('pdftotext', ['-raw', '-f', String(physicalStart), '-l', String(physicalEnd), cfg.pdf, '-'], {
    cwd: ROOT,
    encoding: 'utf8',
    maxBuffer: 1024 * 1024 * 8,
  });
}

function cleanText(value) {
  return String(value || '')
    .replace(/\u0008/g, '')
    .replace(/[“”]/g, '"')
    .replace(/[’]/g, "'")
    .replace(/\s+/g, ' ')
    .replace(/\s+([,.;:])/g, '$1')
    .trim();
}

function splitTopLevelList(text) {
  const items = [];
  let current = '';
  let depth = 0;
  for (const ch of cleanText(text)) {
    if (ch === '(') depth += 1;
    if (ch === ')' && depth > 0) depth -= 1;
    if (ch === ',' && depth === 0) {
      if (current.trim()) items.push(current.trim());
      current = '';
    } else {
      current += ch;
    }
  }
  if (current.trim()) items.push(current.trim());
  return items;
}

function compactList(text, maxItems = 10, maxChars = 260) {
  const items = splitTopLevelList(text)
    .map(item => item.replace(/[.;:\s]+$/, '').trim())
    .filter(Boolean);
  if (!items.length) return '';
  const suffix = items.length > maxItems ? `, +${items.length - maxItems} more` : '';
  const out = items.slice(0, maxItems).join(', ') + suffix;
  return out.length > maxChars ? out.slice(0, maxChars - 1).trimEnd() + '...' : out;
}

function compactText(text, maxChars = 260) {
  const out = cleanText(text).replace(/[.;:\s]+$/, '');
  return out.length > maxChars ? out.slice(0, maxChars - 1).trimEnd() + '...' : out;
}

function section(block, start, stops) {
  const startRe = new RegExp(`${start}\\s*:`, 'i');
  const startMatch = startRe.exec(block);
  if (!startMatch) return '';
  let rest = block.slice(startMatch.index + startMatch[0].length);
  let cut = -1;
  for (const stop of stops) {
    const looseStop = /^(SpecialRules|Using|Ranged Weapons|Melee Weapons|Table|The Headsman|Tides of|Mark of|Cloud of)$/i.test(stop);
    const pattern = looseStop
      ? `(?:^|\\s)${stop}(?:\\s*:|\\b)`
      : `(?:^|\\n)\\s*${stop}\\s*:`;
    const m = new RegExp(pattern, 'i').exec(rest);
    if (m && (cut === -1 || m.index < cut)) cut = m.index;
  }
  if (cut >= 0) rest = rest.slice(0, cut);
  return rest;
}

function profileFromBlock(block) {
  const rows = block.split(/\r?\n/);
  for (const line of rows) {
    const tokens = line.trim().split(/\s+/).filter(Boolean);
    const nums = tokens
      .map(token => token.replace(/[–—-]{1,2}/g, '0'))
      .filter(token => /^\d{1,3}$/.test(token))
      .map(Number);
    if (nums.length >= 9) {
      return Object.fromEntries(PROFILE_KEYS.map((key, idx) => [key, nums[idx]]));
    }
  }
  return null;
}

function armourFromText(text) {
  const cleaned = cleanText(text);
  const all = /\bAll\s+(\d+)/i.exec(cleaned);
  if (all) return { all: Number(all[1]) };

  const body = /\bBody\s+(\d+)/i.exec(cleaned);
  const head = /\bHead\s+(\d+)/i.exec(cleaned);
  const arms = /\bArms?(?:\s+and\s+Legs)?\s+(\d+)/i.exec(cleaned);
  const legs = /\bLegs?\s+(\d+)/i.exec(cleaned);
  if (body || head || arms || legs) {
    return {
      head: head ? Number(head[1]) : (body ? Number(body[1]) : 0),
      body: body ? Number(body[1]) : 0,
      arm: arms ? Number(arms[1]) : (body ? Number(body[1]) : 0),
      leg: legs ? Number(legs[1]) : (arms ? Number(arms[1]) : (body ? Number(body[1]) : 0)),
    };
  }
  return null;
}

function tbFrom(profile, traitsText) {
  const base = Math.floor((Number(profile.t) || 0) / 10);
  const unnatural = /Unnatural Toughness\s*\((?:x|×)(\d+)\)/i.exec(traitsText || '');
  if (unnatural) return base * Number(unnatural[1]);
  const daemonic = /Daemonic\s*\(TB\s*(\d+)\)/i.exec(traitsText || '');
  if (daemonic) return Number(daemonic[1]);
  return base;
}

function parseProfilesFromPage(text) {
  const matches = [];
  const profileRe = new RegExp(`([^\\n\\r]{2,80}?)\\s+\\(${RANK_RE}\\)\\s+Profile`, 'ig');
  let match;
  while ((match = profileRe.exec(text))) {
    matches.push({
      name: cleanText(match[1]).replace(/^[^A-Za-z0-9]+/, ''),
      rank: titleCase(match[2]),
      index: match.index,
    });
  }

  return matches.map((item, idx) => {
    const nextStart = matches[idx + 1]?.index ?? text.length;
    const afterProfile = text.slice(item.index, nextStart);
    const searchableAfterProfile = afterProfile.replace(/\u0008/g, '');
    const profile = profileFromBlock(afterProfile);
    const woundsMatch = /Wounds\s*:?\s*(\d+)/i.exec(searchableAfterProfile);
    if (!profile || !woundsMatch) return null;

    const movementMatch = /\b(?:Movement|Move|Speed)\s*:?\s*([0-9/]+)/i.exec(searchableAfterProfile);
    const skills = compactList(section(searchableAfterProfile, 'Skills', ['Talents', 'Traits', 'Armour', 'Armor', 'Weapons', 'Gear', 'SpecialRules', 'Special']));
    const talents = compactList(section(searchableAfterProfile, 'Talents', ['Traits', 'Armour', 'Armor', 'Weapons', 'Gear', 'SpecialRules', 'Special']));
    const traitsText = section(searchableAfterProfile, 'Traits', ['Armour', 'Armor', 'Weapons', 'Gear', 'SpecialRules', 'Special', 'Psy Rating', 'Psychic Powers']);
    const traits = compactList(traitsText);
    const armourText = section(searchableAfterProfile, 'Armou?r', ['Weapons', 'Gear', 'Special', 'Skills', 'Talents', 'Traits']);
    const weaponsText = section(searchableAfterProfile, 'Weapons', [
      'Gear',
      'SpecialRules',
      'Special',
      'Skills',
      'Talents',
      'Traits',
      'Armour',
      'Armor',
      'Mark of',
      'Cloud of',
      'The Headsman',
      'Tides of',
      'Using',
      'Ranged Weapons',
      'Melee Weapons',
      'Table',
    ]);

    return {
      name: item.name,
      rank: item.rank,
      profile,
      wounds: Number(woundsMatch[1]),
      movement: movementMatch ? movementMatch[1] : '',
      skills,
      talents,
      traits,
      armour: armourFromText(armourText),
      weapons: compactText(weaponsText),
      toughnessBonus: tbFrom(profile, traitsText),
    };
  }).filter(Boolean);
}

function loadBestiary(file) {
  const raw = JSON.parse(fs.readFileSync(file, 'utf8'));
  const arr = Array.isArray(raw) ? raw : (raw.results || raw.entries || raw.items);
  if (!Array.isArray(arr)) throw new Error(`Unsupported bestiary shape in ${file}`);
  return { raw, arr };
}

function toEntry(parsed, indexRow, book) {
  const stats = {
    profile: parsed.profile,
    movement: parsed.movement,
    wounds: parsed.wounds,
    toughnessBonus: parsed.toughnessBonus,
    statSource: 'pdf',
    needsReview: false,
    sourceIndex: '40krpgtools',
    threatTier: parsed.rank,
    parsedName: parsed.name,
  };
  if (parsed.armour) stats.armour = parsed.armour;
  if (parsed.weapons) stats.weapons = parsed.weapons;
  if (parsed.skills) stats.skills = parsed.skills;
  if (parsed.talents) stats.talents = parsed.talents;
  if (parsed.traits) stats.traits = parsed.traits;
  return {
    bestiaryName: indexRow.name,
    type: indexRow.type,
    affiliation: indexRow.affiliation,
    book,
    page: indexRow.page,
    pdf: path.basename(BOOKS[book].pdf),
    sourceUrl: indexRow.sourceUrl,
    wounds: parsed.wounds,
    stats,
  };
}

function main() {
  const book = argValue('book', 'Mark of the Xenos');
  const limit = Number(argValue('limit', '0'));
  const dryRun = hasFlag('dry-run');
  const replaceBook = hasFlag('replace-book');
  if (!BOOKS[book]) throw new Error(`Unsupported book: ${book}`);

  const index = JSON.parse(fs.readFileSync(INDEX_PATH, 'utf8'));
  const { raw, arr } = loadBestiary(PUBLIC_PATH);
  if (replaceBook) {
    for (let i = arr.length - 1; i >= 0; i -= 1) {
      const entry = arr[i] || {};
      if (entry.book === book && entry.stats?.sourceIndex === '40krpgtools') {
        arr.splice(i, 1);
      }
    }
  }
  const existing = new Set(arr.map(entry => norm(entry.bestiaryName || entry.name)).filter(Boolean));
  const rows = index.entries.filter(row => row.book === book && !existing.has(norm(row.name)));

  const pageCache = new Map();
  const parsedByPage = new Map();
  const added = [];
  const skipped = [];

  for (const row of rows) {
    if (limit && added.length >= limit) break;
    if (!pageCache.has(row.page)) {
      const text = textForPage(book, row.page);
      pageCache.set(row.page, text);
      parsedByPage.set(row.page, parseProfilesFromPage(text));
    }
    const parsed = parsedByPage.get(row.page);
    const hit = parsed.find(candidate => {
      const a = norm(candidate.name);
      const b = norm(row.name);
      return a === b || a.includes(b) || b.includes(a);
    });
    if (!hit) {
      skipped.push(`${row.name} (${book} ${row.page})`);
      continue;
    }
    arr.push(toEntry(hit, row, book));
    existing.add(norm(row.name));
    added.push(`${row.name} (${book} ${row.page})`);
  }

  if (!dryRun) {
    if (!Array.isArray(raw)) raw.count = arr.length;
    const serialized = JSON.stringify(raw, null, 2);
    for (const file of [PUBLIC_PATH, DB_PATH]) {
      if (fs.existsSync(file)) fs.writeFileSync(file + '.pre-hydrate.' + Date.now() + '.json', fs.readFileSync(file));
      fs.writeFileSync(file, serialized);
    }
  }

  console.log(`${dryRun ? 'Would add' : 'Added'} ${added.length} entries from ${book}`);
  added.forEach(name => console.log(`  + ${name}`));
  console.log(`Skipped ${skipped.length} rows without a parsed profile match`);
  skipped.slice(0, 20).forEach(name => console.log(`  - ${name}`));
}

main();