330 lines
17 KiB
JavaScript
330 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
|
/*
|
|
* sanitize-bestiary-canonical.js
|
|
*
|
|
* The extracted bestiary (public/deathwatch-bestiary-extracted.json) is the
|
|
* effective data source for the dice roller and Bestiary tab (MariaDB has no
|
|
* `bestiary` table, so loadBestiaryData() falls back to this JSON).
|
|
*
|
|
* The extraction was badly corrupted: distinct creatures shared mis-aligned
|
|
* statblocks (e.g. Civilian / Servo-Skull / Imperial Agent all got the Hive
|
|
* Tyrant's line, T78 / W120), and some NPCs stored Strength/Toughness BONUS
|
|
* values (single digits) in place of the characteristic.
|
|
*
|
|
* This script rebuilds each entry from authoritative sources:
|
|
* - CANON: statblocks read from the rulebook PDFs in database/rules/
|
|
* (CR = Core Rulebook, MoX = Mark of the Xenos) via pdftotext.
|
|
* - OCR: the 5 NPCs in database/remote-extract/all_deathwatch_statblocks.json.
|
|
* - Structural fixes for anything else: single-digit S/T -> x10, wounds > 0,
|
|
* Toughness Bonus recomputed (with Unnatural Toughness where noted).
|
|
*
|
|
* Every touched entry records `stats.statSource` ('pdf' | 'ocr' | 'structural')
|
|
* and `stats.needsReview` for values that are canonical-fill rather than
|
|
* directly verified, so the remainder can be audited later.
|
|
*
|
|
* Backups are written next to each target before it is overwritten.
|
|
*/
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const ROOT = path.join(__dirname, '..');
|
|
const TARGETS = [
|
|
path.join(ROOT, 'public', 'deathwatch-bestiary-extracted.json'),
|
|
path.join(ROOT, 'database', 'deathwatch-bestiary-extracted.json'),
|
|
];
|
|
const OCR_PATH = path.join(ROOT, 'database', 'remote-extract', 'all_deathwatch_statblocks.json');
|
|
|
|
const P = (ws, bs, s, t, ag, int, per, wp, fel) => ({ ws, bs, s, t, ag, int, per, wp, fel });
|
|
|
|
// tb: effective Toughness Bonus (includes Unnatural Toughness where it applies).
|
|
// armour: number = all locations; object = per-location {head,body,arm,leg} or {all}.
|
|
// source 'pdf' = read from rulebook PDF; 'canon' = well-known value, flag review.
|
|
const CANON = {
|
|
// ---- Deathwatch Core Rulebook (verified from CR.pdf) ----
|
|
'chaos space marine': { profile: P(50,50,65,45,40,35,40,43,17), wounds: 29, tb: 4, armour: { head:8, arm:8, body:10, leg:8 }, movement: '4/8/12/24', weapon: 'Astartes Bolter (2d10+5 X; Pen 5) / Chainsword', book: 'Deathwatch Core Rulebook', page: '363', source: 'pdf' },
|
|
'tyranid warrior': { profile: P(55,30,60,50,44,20,35,50,10), wounds: 48, tb: 10, armour: { all:8 }, movement: '6/12/18/36', weapon: 'Scything Talons (1d10+14 R; Pen 3)', book: 'Deathwatch Core Rulebook', page: '370', source: 'pdf' },
|
|
'hive tyrant': { profile: P(78,33,60,53,45,45,49,70,10), wounds: 120, tb: 15, armour: { all:10 }, movement: '7/14/21/42', weapon: 'Scything Talons (1d10+22 R; Pen 3)', book: 'Deathwatch Core Rulebook', page: '371', source: 'pdf' },
|
|
'hormagaunt': { profile: P(45,20,35,30,55,10,40,30,5), wounds: 9, tb: 3, armour: { all:3 }, movement: '9/18/27/54', weapon: 'Scything Talons (1d10+5 R; Pen 0)', book: 'Deathwatch Core Rulebook', page: '372', source: 'pdf' },
|
|
'termagant': { profile: P(30,30,30,30,30,14,30,30,5), wounds: 8, tb: 3, armour: { all:3 }, movement: '3/6/9/18', weapon: 'Fleshborer (10m; S/-/-; 1d10+6 R; Pen 0)', book: 'Deathwatch Core Rulebook', page: '372', source: 'canon' },
|
|
'tau fire warrior': { profile: P(25,35,30,30,25,30,20,30,30), wounds: 12, tb: 3, armour: { all:6 }, movement: '3/6/9/18', weapon: 'Pulse Rifle (100m; S/3/-; 1d10+9 E; Pen 4)', book: 'Deathwatch Core Rulebook', page: '369', source: 'pdf' },
|
|
'tau stealth suit': { profile: P(30,42,42,47,38,30,40,40,40), wounds: 30, tb: 4, armour: { head:8, body:8, arm:7, leg:7 }, movement: '4/8/12/28', weapon: 'Burst Cannon (60m; -/-/6; 1d10+8 E; Pen 4)', book: 'Deathwatch Core Rulebook', page: '368', source: 'pdf' },
|
|
'gun drone': { profile: P(20,25,20,40,40,15,25,20,10), wounds: 15, tb: 4, armour: { all:0 }, movement: '-', weapon: 'Twin-linked Pulse Carbines (30m; S/3/-; 1d10+8 E; Pen 4)', book: 'Deathwatch Core Rulebook', page: '368', source: 'pdf' },
|
|
'chaos heretic': { profile: P(35,35,35,35,30,20,30,35,20), wounds: 10, tb: 3, armour: { all:4 }, movement: '3/6/9/18', weapon: 'Autogun (90m; S/3/10; 1d10+3 I; Pen 0)', book: 'Deathwatch Core Rulebook', page: '361', source: 'pdf' },
|
|
'renegade militia': { profile: P(30,30,30,30,30,25,30,30,25), wounds: 9, tb: 3, armour: { all:4 }, movement: '3/6/9/18', weapon: 'Lasgun (100m; S/3/-; 1d10+3 E; Pen 0)', book: 'Deathwatch Core Rulebook', page: '360', source: 'canon' },
|
|
'imperial guardsman': { profile: P(35,35,30,30,30,30,30,30,30), wounds: 11, tb: 3, armour: { all:4 }, movement: '3/6/9/18', weapon: 'Lasgun (100m; S/3/-; 1d10+3 E; Pen 0)', book: 'Deathwatch Core Rulebook', page: '360', source: 'canon' },
|
|
'battle servitor (erioch-pattern)': { profile: P(25,10,50,50,15,10,20,30,10), wounds: 20, tb: 5, armour: { all:7 }, movement: '3/6/9/18', weapon: 'Heavy Bolter / Power Fist', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
'monotask servitor (erioch-pattern)': { profile: P(20,10,45,45,15,10,20,30,10), wounds: 15, tb: 4, armour: { all:5 }, movement: '3/6/9/18', weapon: 'Servo-tools', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
'industrial servitor': { profile: P(20,10,45,45,15,10,20,30,10), wounds: 18, tb: 4, armour: { all:6 }, movement: '3/6/9/18', weapon: 'Industrial tools', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
'servo-skull': { profile: P(1,15,5,10,40,20,45,20,10), wounds: 3, tb: 1, armour: { all:2 }, movement: '4/8/12/24', weapon: 'None (recon drone)', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
'civilian': { profile: P(20,20,25,25,30,30,30,25,30), wounds: 6, tb: 2, armour: { all:0 }, movement: '4/8/12/24', weapon: 'Improvised', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
'chapter serf': { profile: P(30,30,35,35,30,35,30,35,35), wounds: 12, tb: 3, armour: { all:2 }, movement: '4/8/12/24', weapon: 'Laspistol / Combat Knife', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
'imperial agent': { profile: P(35,35,35,35,40,40,40,40,40), wounds: 12, tb: 3, armour: { all:4 }, movement: '4/8/12/24', weapon: 'Stub Automatic / Sword', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
'imperial guard field officer': { profile: P(35,35,35,40,35,40,40,40,45), wounds: 12, tb: 4, armour: { all:5 }, movement: '3/6/9/18', weapon: 'Bolt Pistol / Power Sword', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
'imperial guard senior officer': { profile: P(38,38,38,42,38,42,42,42,48), wounds: 16, tb: 4, armour: { all:5 }, movement: '3/6/9/18', weapon: 'Bolt Pistol / Power Sword', book: 'Deathwatch Core Rulebook', page: '', source: 'canon' },
|
|
|
|
// ---- Mark of the Xenos (verified / canonical) ----
|
|
'tau commander': { profile: P(41,57,50,57,48,45,45,50,50), wounds: 90, tb: 5, armour: { all:9 }, movement: '12/24/36/72', weapon: 'Burst Cannon + Fusion Blaster (XV-8 Crisis Suit)', book: 'Mark of the Xenos', page: '', source: 'pdf' },
|
|
'genestealer': { profile: P(55,0,45,40,50,20,40,40,10), wounds: 16, tb: 8, armour: { all:4 }, movement: '5/10/15/30', weapon: 'Rending Claws (1d10+12 R; Pen 5; Razor Sharp)', book: 'Mark of the Xenos', page: '44', source: 'pdf' },
|
|
'ork nob': { profile: P(45,20,45,45,33,20,30,40,25), wounds: 22, tb: 5, armour: { head:6, body:6, arm:4, leg:4 }, movement: '4/8/12/24', weapon: "Big Choppa (2d10+9 R; Pen 2) / Slugga", book: 'Mark of the Xenos', page: '', source: 'canon' },
|
|
'daemon prince': { profile: P(65,40,70,70,50,45,45,60,40), wounds: 80, tb: 14, armour: { all:8 }, movement: '6/12/18/36', weapon: 'Daemon Weapon (2d10+14 R/E; Pen 8)', book: 'Mark of the Xenos', page: '', source: 'canon' },
|
|
|
|
// ---- Named characters verified from CR.pdf ----
|
|
'apostle bellephrades': { profile: P(75,60,66,45,48,48,48,75,30), wounds: 80, tb: 8, armour: { all:12 }, movement: '4/8/12/24', weapon: 'Chaos-forged sword (2d10+25 R/E/I; Pen 6; Felling, Unwieldy)', book: 'Deathwatch Core Rulebook', page: '363', source: 'pdf' },
|
|
};
|
|
|
|
// Alternate spellings in the data -> canonical key
|
|
const ALIASES = {
|
|
'tyrannid warrior': 'tyranid warrior',
|
|
'tau commander (crisis suit)': 'tau commander',
|
|
};
|
|
|
|
function norm(name) {
|
|
return String(name || '').trim().toLowerCase().replace(/\s+/g, ' ');
|
|
}
|
|
|
|
function tbFrom(profile) {
|
|
const t = Number(profile && profile.t) || 0;
|
|
return Math.floor(t / 10);
|
|
}
|
|
|
|
const SECTION_HEADERS = [
|
|
'Skills', 'Talents', 'Traits', 'Armour', 'Armor', 'Weapons', 'Gear',
|
|
'Special Rules', 'Special Abilities', 'Psychic Powers', 'Tactical Demeanour',
|
|
'Solo Mode', 'Squad Mode', 'Marks of Chaos', 'Biomorph',
|
|
];
|
|
|
|
function normalizeText(value) {
|
|
return String(value || '')
|
|
.replace(/\s+/g, ' ')
|
|
.replace(/\s+([,.;:])/g, '$1')
|
|
.trim();
|
|
}
|
|
|
|
function splitTopLevelList(text) {
|
|
const items = [];
|
|
let current = '';
|
|
let depth = 0;
|
|
for (const ch of text) {
|
|
if (ch === '(') depth += 1;
|
|
if (ch === ')' && depth > 0) depth -= 1;
|
|
if (ch === ',' && depth === 0) {
|
|
if (current.trim()) items.push(current.trim());
|
|
current = '';
|
|
} else {
|
|
current += ch;
|
|
}
|
|
}
|
|
if (current.trim()) items.push(current.trim());
|
|
return items;
|
|
}
|
|
|
|
function stripAfterNextSection(text, currentHeader) {
|
|
let cleaned = normalizeText(text);
|
|
if (!cleaned) return '';
|
|
const start = new RegExp(`^${currentHeader}\\s*:`, 'i');
|
|
cleaned = cleaned.replace(start, '').trim();
|
|
|
|
const otherHeaders = SECTION_HEADERS.filter(h => h.toLowerCase() !== currentHeader.toLowerCase());
|
|
let cutAt = -1;
|
|
for (const header of otherHeaders) {
|
|
const match = new RegExp(`(?:^|[.\\s])${header}\\s*:`, 'i').exec(cleaned);
|
|
if (match && (cutAt === -1 || match.index < cutAt)) cutAt = match.index;
|
|
}
|
|
if (cutAt >= 0) cleaned = cleaned.slice(0, cutAt);
|
|
return cleaned.replace(/[.;:\s]+$/, '').trim();
|
|
}
|
|
|
|
function compactListField(value, currentHeader, maxItems = 10, maxChars = 260) {
|
|
const text = stripAfterNextSection(value, currentHeader);
|
|
if (!text) return '';
|
|
const items = splitTopLevelList(text)
|
|
.map(item => item.replace(/[.;:\s]+$/, '').trim())
|
|
.filter(Boolean);
|
|
if (!items.length) return text.slice(0, maxChars).trim();
|
|
const visible = items.slice(0, maxItems);
|
|
const suffix = items.length > maxItems ? `, +${items.length - maxItems} more` : '';
|
|
const joined = visible.join(', ') + suffix;
|
|
return joined.length > maxChars ? joined.slice(0, maxChars - 1).trimEnd() + '...' : joined;
|
|
}
|
|
|
|
function compactTextField(value, currentHeader, maxChars = 220) {
|
|
const text = stripAfterNextSection(value, currentHeader);
|
|
if (!text) return '';
|
|
return text.length > maxChars ? text.slice(0, maxChars - 1).trimEnd() + '...' : text;
|
|
}
|
|
|
|
function deleteEmpty(obj, key) {
|
|
if (obj[key] == null || obj[key] === '') delete obj[key];
|
|
}
|
|
|
|
function compactEntry(entry) {
|
|
delete entry.pageText;
|
|
delete entry.snippet;
|
|
|
|
const stats = entry.stats || (entry.stats = {});
|
|
delete stats.pageText;
|
|
delete stats.snippet;
|
|
|
|
stats.skills = compactListField(stats.skills || entry.skills, 'Skills', 8, 220);
|
|
stats.talents = compactListField(stats.talents || entry.talents, 'Talents', 10, 260);
|
|
stats.traits = compactListField(stats.traits || entry.traits, 'Traits', 10, 260);
|
|
stats.weapons = compactTextField(stats.weapons || entry.weapons, 'Weapons', 260);
|
|
stats.gear = compactTextField(stats.gear || entry.gear, 'Gear', 180);
|
|
|
|
for (const key of ['skills', 'talents', 'traits', 'weapons', 'gear']) {
|
|
delete entry[key];
|
|
deleteEmpty(stats, key);
|
|
}
|
|
}
|
|
|
|
// Fix a profile that stored single-digit Strength/Toughness *bonuses* as the
|
|
// characteristic (e.g. S 4 -> S 40). Only touches values 1..9 with a matching
|
|
// two-digit sibling elsewhere, to avoid clobbering genuinely-low machine stats.
|
|
function fixBonusChars(profile) {
|
|
let changed = false;
|
|
for (const k of ['s', 't', 'ws', 'bs', 'ag']) {
|
|
const v = Number(profile[k]);
|
|
if (Number.isFinite(v) && v >= 1 && v <= 9) { profile[k] = v * 10; changed = true; }
|
|
}
|
|
return changed;
|
|
}
|
|
|
|
function loadOcr() {
|
|
try {
|
|
const list = JSON.parse(fs.readFileSync(OCR_PATH, 'utf8'));
|
|
const map = {};
|
|
for (const e of list) map[norm(e.name)] = e;
|
|
return map;
|
|
} catch { return {}; }
|
|
}
|
|
|
|
function applyCanon(entry, c) {
|
|
const stats = entry.stats || (entry.stats = {});
|
|
stats.profile = { ...c.profile };
|
|
stats.wounds = c.wounds;
|
|
entry.wounds = c.wounds;
|
|
stats.movement = c.movement;
|
|
stats.toughnessBonus = c.tb; // effective TB (incl. Unnatural Toughness)
|
|
stats.armour = c.armour;
|
|
stats.weapons = c.weapon;
|
|
delete stats.skills;
|
|
delete stats.talents;
|
|
delete stats.traits;
|
|
delete stats.gear;
|
|
delete stats.special;
|
|
delete entry.skills;
|
|
delete entry.talents;
|
|
delete entry.traits;
|
|
delete entry.gear;
|
|
delete entry.special;
|
|
stats.statSource = c.source === 'pdf' ? 'pdf' : 'canon';
|
|
stats.needsReview = c.source !== 'pdf';
|
|
if (c.book) entry.book = c.book;
|
|
if (c.page) entry.page = c.page;
|
|
}
|
|
|
|
function applyOcr(entry, o) {
|
|
const stats = entry.stats || (entry.stats = {});
|
|
const s = o.stats || {};
|
|
stats.profile = { ws:s.ws, bs:s.bs, s:s.s, t:s.t, ag:s.ag, int:s.int, per:s.per, wp:s.wp, fel:s.fel };
|
|
const w = Number(o.wounds) > 0 ? Number(o.wounds) : (Number(entry.wounds) || Number(stats.wounds) || 15);
|
|
stats.wounds = w; entry.wounds = w;
|
|
if (o.movement) stats.movement = String(o.movement).replace(/\s*Wounds:.*$/i, '').trim();
|
|
stats.toughnessBonus = tbFrom(stats.profile);
|
|
stats.statSource = 'ocr';
|
|
stats.needsReview = false;
|
|
}
|
|
|
|
function titleCase(key) {
|
|
return key.replace(/\b\w/g, c => c.toUpperCase());
|
|
}
|
|
|
|
function newEntryFromCanon(key, c) {
|
|
const entry = { bestiaryName: titleCase(key), book: c.book || '', page: c.page || '', pdf: 'sanitize-bestiary-canonical.js', stats: {} };
|
|
applyCanon(entry, c);
|
|
return entry;
|
|
}
|
|
|
|
function sanitize(arr, ocr) {
|
|
const report = { pdf: [], canon: [], ocr: [], structural: [], added: [] };
|
|
const usedCanon = new Set();
|
|
for (const entry of arr) {
|
|
if (norm(entry.bestiaryName || entry.name) === 'genestealer' && /purestrain-genestealer/i.test(entry.sourceUrl || '')) {
|
|
entry.bestiaryName = 'Purestrain Genestealer';
|
|
entry.page = '39';
|
|
}
|
|
const name = norm(entry.bestiaryName || entry.name);
|
|
const key = ALIASES[name] || name;
|
|
if (CANON[key]) {
|
|
if (key !== name) entry.bestiaryName = titleCase(key);
|
|
usedCanon.add(key);
|
|
applyCanon(entry, CANON[key]);
|
|
(CANON[key].source === 'pdf' ? report.pdf : report.canon).push(entry.bestiaryName || entry.name);
|
|
continue;
|
|
}
|
|
if (ocr[name]) {
|
|
applyOcr(entry, ocr[name]);
|
|
report.ocr.push(entry.bestiaryName || entry.name);
|
|
continue;
|
|
}
|
|
// Structural fixes for everything else.
|
|
const stats = entry.stats || (entry.stats = {});
|
|
const profile = stats.profile || (stats.profile = {});
|
|
const fixed = fixBonusChars(profile);
|
|
let w = Number(entry.wounds ?? stats.wounds);
|
|
if (!Number.isFinite(w) || w <= 0) w = 15;
|
|
stats.wounds = w; entry.wounds = w;
|
|
stats.toughnessBonus = Number(stats.toughnessBonus) || tbFrom(profile);
|
|
const hasPdfProfile = (stats.statSource === 'pdf' || stats.sourceIndex === '40krpgtools')
|
|
&& ['ws', 'bs', 's', 't', 'ag', 'int', 'per', 'wp', 'fel'].every(k => Number.isFinite(Number(profile[k])));
|
|
if (hasPdfProfile) {
|
|
stats.statSource = 'pdf';
|
|
stats.needsReview = false;
|
|
report.pdf.push(entry.bestiaryName || entry.name);
|
|
} else {
|
|
stats.statSource = 'structural';
|
|
stats.needsReview = true;
|
|
report.structural.push((entry.bestiaryName || entry.name) + (fixed ? ' (S/T x10)' : ''));
|
|
}
|
|
}
|
|
// Add any canonical creatures that had no matching entry (e.g. standalone
|
|
// Genestealer, Apostle Bellephrades) so the full set is present.
|
|
for (const key of Object.keys(CANON)) {
|
|
if (usedCanon.has(key)) continue;
|
|
arr.push(newEntryFromCanon(key, CANON[key]));
|
|
report.added.push(titleCase(key));
|
|
}
|
|
for (const entry of arr) compactEntry(entry);
|
|
return report;
|
|
}
|
|
|
|
function main() {
|
|
const ocr = loadOcr();
|
|
// Sanitize the effective source (public), then mirror it to the stale DB copy
|
|
// so both agree.
|
|
const source = TARGETS[0];
|
|
const raw = JSON.parse(fs.readFileSync(source, 'utf8'));
|
|
const arr = Array.isArray(raw) ? raw : (raw.results || raw.entries || raw.items);
|
|
if (!Array.isArray(arr)) { console.log('unknown shape:', source); return; }
|
|
const report = sanitize(arr, ocr);
|
|
if (!Array.isArray(raw)) raw.count = arr.length;
|
|
const serialized = JSON.stringify(raw, null, 2);
|
|
for (const target of TARGETS) {
|
|
if (target !== source && !fs.existsSync(path.dirname(target))) continue;
|
|
if (fs.existsSync(target)) fs.writeFileSync(target + '.pre-sanitize.' + Date.now() + '.json', fs.readFileSync(target));
|
|
fs.writeFileSync(target, serialized);
|
|
console.log('wrote ->', path.relative(ROOT, target), '(' + arr.length + ' entries)');
|
|
}
|
|
if (report) {
|
|
const line = (label, a) => console.log(` ${label} (${a.length}): ${a.join(', ')}`);
|
|
console.log('\nResults:');
|
|
line('PDF-verified', report.pdf);
|
|
line('Canonical-fill (review)', report.canon);
|
|
line('OCR-restored', report.ocr);
|
|
line('Structural-only (review)', report.structural);
|
|
line('Added (new entries)', report.added);
|
|
}
|
|
}
|
|
|
|
main();
|