dwroller/scripts/import-40krpgtools-bestiary-index.js

#!/usr/bin/env node
/*
 * Import the public 40k RPG Tools master bestiary index into a staging file.
 *
 * The source page is an index of names, type, affiliation, setting, book, and
 * page numbers. It is not a statblock source, so this script intentionally does
 * not add rows to the live bestiary JSON. Use the staged output to decide which
 * local PDFs/pages to hydrate into compact verified entries.
 */
const fs = require('fs');
const path = require('path');
const https = require('https');
if (typeof global.File === 'undefined') {
  global.File = class File {};
}
const cheerio = require('cheerio');

const ROOT = path.join(__dirname, '..');
const SOURCE_URL = 'https://www.40krpgtools.com/bestiary/';
const OUT_PATH = path.join(ROOT, 'database', 'remote-extract', '40krpgtools-bestiary-index.json');
const ACTIVE_BESTIARY = path.join(ROOT, 'public', 'deathwatch-bestiary-extracted.json');

function argValue(name, fallback = null) {
  const prefix = `--${name}=`;
  const found = process.argv.find(arg => arg.startsWith(prefix));
  return found ? found.slice(prefix.length) : fallback;
}

function hasFlag(name) {
  return process.argv.includes(`--${name}`);
}

function norm(value) {
  return String(value || '').trim().toLowerCase().replace(/\s+/g, ' ');
}

function fetchText(url) {
  return new Promise((resolve, reject) => {
    https.get(url, response => {
      if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
        fetchText(new URL(response.headers.location, url).toString()).then(resolve, reject);
        return;
      }
      if (response.statusCode !== 200) {
        reject(new Error(`GET ${url} failed with HTTP ${response.statusCode}`));
        return;
      }
      let body = '';
      response.setEncoding('utf8');
      response.on('data', chunk => { body += chunk; });
      response.on('end', () => resolve(body));
    }).on('error', reject);
  });
}

function loadActiveNames() {
  try {
    const raw = JSON.parse(fs.readFileSync(ACTIVE_BESTIARY, 'utf8'));
    const arr = Array.isArray(raw) ? raw : (raw.results || raw.entries || raw.items || []);
    return new Set(arr.map(entry => norm(entry.bestiaryName || entry.name)).filter(Boolean));
  } catch {
    return new Set();
  }
}

function parseRows(html, activeNames) {
  const $ = cheerio.load(html);
  const rows = [];
  $('#bestiaryTable tr').each((_, tr) => {
    const cells = $(tr).find('td');
    if (cells.length < 6) return;
    const link = $(cells[0]).find('a').first();
    const href = link.attr('href') || '';
    const entry = {
      name: link.text().trim() || $(cells[0]).text().trim(),
      type: $(cells[1]).text().trim(),
      affiliation: $(cells[2]).text().trim(),
      setting: $(cells[3]).text().trim(),
      book: $(cells[4]).text().trim(),
      page: $(cells[5]).text().trim(),
      sourceUrl: href ? new URL(href, SOURCE_URL).toString() : SOURCE_URL,
    };
    entry.key = norm(entry.name);
    entry.inActiveBestiary = activeNames.has(entry.key);
    rows.push(entry);
  });
  return rows;
}

async function main() {
  const setting = argValue('setting', 'Deathwatch');
  const includeExisting = hasFlag('include-existing');
  const allSettings = hasFlag('all-settings');
  const activeNames = loadActiveNames();
  const html = await fetchText(SOURCE_URL);
  let rows = parseRows(html, activeNames);

  if (!allSettings) rows = rows.filter(row => norm(row.setting) === norm(setting));
  if (!includeExisting) rows = rows.filter(row => !row.inActiveBestiary);

  rows.sort((a, b) => (
    a.book.localeCompare(b.book) ||
    Number(a.page || 0) - Number(b.page || 0) ||
    a.name.localeCompare(b.name)
  ));

  const byBook = rows.reduce((acc, row) => {
    acc[row.book] = (acc[row.book] || 0) + 1;
    return acc;
  }, {});

  const output = {
    source: SOURCE_URL,
    importedAt: new Date().toISOString(),
    mode: {
      setting: allSettings ? 'all' : setting,
      includeExisting,
    },
    count: rows.length,
    byBook,
    entries: rows,
  };

  fs.mkdirSync(path.dirname(OUT_PATH), { recursive: true });
  fs.writeFileSync(OUT_PATH, JSON.stringify(output, null, 2));

  console.log(`Wrote ${rows.length} staged index rows -> ${path.relative(ROOT, OUT_PATH)}`);
  Object.entries(byBook)
    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
    .forEach(([book, count]) => console.log(`  ${book}: ${count}`));
}

main().catch(error => {
  console.error(error);
  process.exit(1);
});