Fix bestiary data: sync profiles, armour, movement from stats; fix cross-contaminated entries
This commit is contained in:
327
scripts/hydrate-bestiary-from-40krpgtools.js
Normal file
327
scripts/hydrate-bestiary-from-40krpgtools.js
Normal file
@@ -0,0 +1,327 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* Hydrate staged 40k RPG Tools index rows from local PDFs into the compact live
|
||||
* bestiary JSON. This does not scrape statblocks from the web; it uses the web
|
||||
* index as a page map and extracts only compact combat fields from local PDFs.
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execFileSync } = require('child_process');
|
||||
|
||||
const ROOT = path.join(__dirname, '..');
|
||||
const INDEX_PATH = path.join(ROOT, 'database', 'remote-extract', '40krpgtools-bestiary-index.json');
|
||||
const PUBLIC_PATH = path.join(ROOT, 'public', 'deathwatch-bestiary-extracted.json');
|
||||
const DB_PATH = path.join(ROOT, 'database', 'deathwatch-bestiary-extracted.json');
|
||||
|
||||
const BOOKS = {
|
||||
'Mark of the Xenos': { pdf: path.join(ROOT, 'database', 'rules', 'MoX.pdf'), pageOffset: 6 },
|
||||
'First Founding': { pdf: path.join(ROOT, 'database', 'rules', 'FF.pdf'), pageOffset: 8 },
|
||||
'The Emperor Protects': { pdf: path.join(ROOT, 'database', 'rules', 'HtC.pdf'), pageOffset: 8 },
|
||||
'Rites of Battle': { pdf: path.join(ROOT, 'database', 'rules', 'RoB.pdf'), pageOffset: 8 },
|
||||
'Deathwatch Core Rulebook': { pdf: path.join(ROOT, 'database', 'rules', 'CR.pdf'), pageOffset: 0 },
|
||||
};
|
||||
|
||||
const PROFILE_KEYS = ['ws', 'bs', 's', 't', 'ag', 'int', 'per', 'wp', 'fel'];
|
||||
const RANK_RE = '(Troops?|Elite|Master|Minion|Personal|Vehicle|Horde)';
|
||||
|
||||
function argValue(name, fallback = null) {
|
||||
const prefix = `--${name}=`;
|
||||
const found = process.argv.find(arg => arg.startsWith(prefix));
|
||||
return found ? found.slice(prefix.length) : fallback;
|
||||
}
|
||||
|
||||
function hasFlag(name) {
|
||||
return process.argv.includes(`--${name}`);
|
||||
}
|
||||
|
||||
function norm(value) {
|
||||
return String(value || '')
|
||||
.toLowerCase()
|
||||
.replace(/[’']/g, '')
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.trim()
|
||||
.replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
function titleCase(value) {
|
||||
return String(value || '').replace(/\b\w/g, c => c.toUpperCase());
|
||||
}
|
||||
|
||||
function textForPage(book, page) {
|
||||
const cfg = BOOKS[book];
|
||||
const listedPage = Number(page);
|
||||
const physicalStart = Math.max(1, listedPage);
|
||||
const physicalEnd = listedPage + cfg.pageOffset + 2;
|
||||
return execFileSync('pdftotext', ['-raw', '-f', String(physicalStart), '-l', String(physicalEnd), cfg.pdf, '-'], {
|
||||
cwd: ROOT,
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 1024 * 1024 * 8,
|
||||
});
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
return String(value || '')
|
||||
.replace(/\u0008/g, '')
|
||||
.replace(/[“”]/g, '"')
|
||||
.replace(/[’]/g, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/\s+([,.;:])/g, '$1')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function splitTopLevelList(text) {
|
||||
const items = [];
|
||||
let current = '';
|
||||
let depth = 0;
|
||||
for (const ch of cleanText(text)) {
|
||||
if (ch === '(') depth += 1;
|
||||
if (ch === ')' && depth > 0) depth -= 1;
|
||||
if (ch === ',' && depth === 0) {
|
||||
if (current.trim()) items.push(current.trim());
|
||||
current = '';
|
||||
} else {
|
||||
current += ch;
|
||||
}
|
||||
}
|
||||
if (current.trim()) items.push(current.trim());
|
||||
return items;
|
||||
}
|
||||
|
||||
function compactList(text, maxItems = 10, maxChars = 260) {
|
||||
const items = splitTopLevelList(text)
|
||||
.map(item => item.replace(/[.;:\s]+$/, '').trim())
|
||||
.filter(Boolean);
|
||||
if (!items.length) return '';
|
||||
const suffix = items.length > maxItems ? `, +${items.length - maxItems} more` : '';
|
||||
const out = items.slice(0, maxItems).join(', ') + suffix;
|
||||
return out.length > maxChars ? out.slice(0, maxChars - 1).trimEnd() + '...' : out;
|
||||
}
|
||||
|
||||
function compactText(text, maxChars = 260) {
|
||||
const out = cleanText(text).replace(/[.;:\s]+$/, '');
|
||||
return out.length > maxChars ? out.slice(0, maxChars - 1).trimEnd() + '...' : out;
|
||||
}
|
||||
|
||||
function section(block, start, stops) {
|
||||
const startRe = new RegExp(`${start}\\s*:`, 'i');
|
||||
const startMatch = startRe.exec(block);
|
||||
if (!startMatch) return '';
|
||||
let rest = block.slice(startMatch.index + startMatch[0].length);
|
||||
let cut = -1;
|
||||
for (const stop of stops) {
|
||||
const looseStop = /^(SpecialRules|Using|Ranged Weapons|Melee Weapons|Table|The Headsman|Tides of|Mark of|Cloud of)$/i.test(stop);
|
||||
const pattern = looseStop
|
||||
? `(?:^|\\s)${stop}(?:\\s*:|\\b)`
|
||||
: `(?:^|\\n)\\s*${stop}\\s*:`;
|
||||
const m = new RegExp(pattern, 'i').exec(rest);
|
||||
if (m && (cut === -1 || m.index < cut)) cut = m.index;
|
||||
}
|
||||
if (cut >= 0) rest = rest.slice(0, cut);
|
||||
return rest;
|
||||
}
|
||||
|
||||
function profileFromBlock(block) {
|
||||
const rows = block.split(/\r?\n/);
|
||||
for (const line of rows) {
|
||||
const tokens = line.trim().split(/\s+/).filter(Boolean);
|
||||
const nums = tokens
|
||||
.map(token => token.replace(/[–—-]{1,2}/g, '0'))
|
||||
.filter(token => /^\d{1,3}$/.test(token))
|
||||
.map(Number);
|
||||
if (nums.length >= 9) {
|
||||
return Object.fromEntries(PROFILE_KEYS.map((key, idx) => [key, nums[idx]]));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function armourFromText(text) {
|
||||
const cleaned = cleanText(text);
|
||||
const all = /\bAll\s+(\d+)/i.exec(cleaned);
|
||||
if (all) return { all: Number(all[1]) };
|
||||
|
||||
const body = /\bBody\s+(\d+)/i.exec(cleaned);
|
||||
const head = /\bHead\s+(\d+)/i.exec(cleaned);
|
||||
const arms = /\bArms?(?:\s+and\s+Legs)?\s+(\d+)/i.exec(cleaned);
|
||||
const legs = /\bLegs?\s+(\d+)/i.exec(cleaned);
|
||||
if (body || head || arms || legs) {
|
||||
return {
|
||||
head: head ? Number(head[1]) : (body ? Number(body[1]) : 0),
|
||||
body: body ? Number(body[1]) : 0,
|
||||
arm: arms ? Number(arms[1]) : (body ? Number(body[1]) : 0),
|
||||
leg: legs ? Number(legs[1]) : (arms ? Number(arms[1]) : (body ? Number(body[1]) : 0)),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function tbFrom(profile, traitsText) {
|
||||
const base = Math.floor((Number(profile.t) || 0) / 10);
|
||||
const unnatural = /Unnatural Toughness\s*\((?:x|×)(\d+)\)/i.exec(traitsText || '');
|
||||
if (unnatural) return base * Number(unnatural[1]);
|
||||
const daemonic = /Daemonic\s*\(TB\s*(\d+)\)/i.exec(traitsText || '');
|
||||
if (daemonic) return Number(daemonic[1]);
|
||||
return base;
|
||||
}
|
||||
|
||||
function parseProfilesFromPage(text) {
|
||||
const matches = [];
|
||||
const profileRe = new RegExp(`([^\\n\\r]{2,80}?)\\s+\\(${RANK_RE}\\)\\s+Profile`, 'ig');
|
||||
let match;
|
||||
while ((match = profileRe.exec(text))) {
|
||||
matches.push({
|
||||
name: cleanText(match[1]).replace(/^[^A-Za-z0-9]+/, ''),
|
||||
rank: titleCase(match[2]),
|
||||
index: match.index,
|
||||
});
|
||||
}
|
||||
|
||||
return matches.map((item, idx) => {
|
||||
const nextStart = matches[idx + 1]?.index ?? text.length;
|
||||
const afterProfile = text.slice(item.index, nextStart);
|
||||
const searchableAfterProfile = afterProfile.replace(/\u0008/g, '');
|
||||
const profile = profileFromBlock(afterProfile);
|
||||
const woundsMatch = /Wounds\s*:?\s*(\d+)/i.exec(searchableAfterProfile);
|
||||
if (!profile || !woundsMatch) return null;
|
||||
|
||||
const movementMatch = /\b(?:Movement|Move|Speed)\s*:?\s*([0-9/]+)/i.exec(searchableAfterProfile);
|
||||
const skills = compactList(section(searchableAfterProfile, 'Skills', ['Talents', 'Traits', 'Armour', 'Armor', 'Weapons', 'Gear', 'SpecialRules', 'Special']));
|
||||
const talents = compactList(section(searchableAfterProfile, 'Talents', ['Traits', 'Armour', 'Armor', 'Weapons', 'Gear', 'SpecialRules', 'Special']));
|
||||
const traitsText = section(searchableAfterProfile, 'Traits', ['Armour', 'Armor', 'Weapons', 'Gear', 'SpecialRules', 'Special', 'Psy Rating', 'Psychic Powers']);
|
||||
const traits = compactList(traitsText);
|
||||
const armourText = section(searchableAfterProfile, 'Armou?r', ['Weapons', 'Gear', 'Special', 'Skills', 'Talents', 'Traits']);
|
||||
const weaponsText = section(searchableAfterProfile, 'Weapons', [
|
||||
'Gear',
|
||||
'SpecialRules',
|
||||
'Special',
|
||||
'Skills',
|
||||
'Talents',
|
||||
'Traits',
|
||||
'Armour',
|
||||
'Armor',
|
||||
'Mark of',
|
||||
'Cloud of',
|
||||
'The Headsman',
|
||||
'Tides of',
|
||||
'Using',
|
||||
'Ranged Weapons',
|
||||
'Melee Weapons',
|
||||
'Table',
|
||||
]);
|
||||
|
||||
return {
|
||||
name: item.name,
|
||||
rank: item.rank,
|
||||
profile,
|
||||
wounds: Number(woundsMatch[1]),
|
||||
movement: movementMatch ? movementMatch[1] : '',
|
||||
skills,
|
||||
talents,
|
||||
traits,
|
||||
armour: armourFromText(armourText),
|
||||
weapons: compactText(weaponsText),
|
||||
toughnessBonus: tbFrom(profile, traitsText),
|
||||
};
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
function loadBestiary(file) {
|
||||
const raw = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
const arr = Array.isArray(raw) ? raw : (raw.results || raw.entries || raw.items);
|
||||
if (!Array.isArray(arr)) throw new Error(`Unsupported bestiary shape in ${file}`);
|
||||
return { raw, arr };
|
||||
}
|
||||
|
||||
function toEntry(parsed, indexRow, book) {
|
||||
const stats = {
|
||||
profile: parsed.profile,
|
||||
movement: parsed.movement,
|
||||
wounds: parsed.wounds,
|
||||
toughnessBonus: parsed.toughnessBonus,
|
||||
statSource: 'pdf',
|
||||
needsReview: false,
|
||||
sourceIndex: '40krpgtools',
|
||||
threatTier: parsed.rank,
|
||||
parsedName: parsed.name,
|
||||
};
|
||||
if (parsed.armour) stats.armour = parsed.armour;
|
||||
if (parsed.weapons) stats.weapons = parsed.weapons;
|
||||
if (parsed.skills) stats.skills = parsed.skills;
|
||||
if (parsed.talents) stats.talents = parsed.talents;
|
||||
if (parsed.traits) stats.traits = parsed.traits;
|
||||
return {
|
||||
bestiaryName: indexRow.name,
|
||||
type: indexRow.type,
|
||||
affiliation: indexRow.affiliation,
|
||||
book,
|
||||
page: indexRow.page,
|
||||
pdf: path.basename(BOOKS[book].pdf),
|
||||
sourceUrl: indexRow.sourceUrl,
|
||||
wounds: parsed.wounds,
|
||||
stats,
|
||||
};
|
||||
}
|
||||
|
||||
function main() {
|
||||
const book = argValue('book', 'Mark of the Xenos');
|
||||
const limit = Number(argValue('limit', '0'));
|
||||
const dryRun = hasFlag('dry-run');
|
||||
const replaceBook = hasFlag('replace-book');
|
||||
if (!BOOKS[book]) throw new Error(`Unsupported book: ${book}`);
|
||||
|
||||
const index = JSON.parse(fs.readFileSync(INDEX_PATH, 'utf8'));
|
||||
const { raw, arr } = loadBestiary(PUBLIC_PATH);
|
||||
if (replaceBook) {
|
||||
for (let i = arr.length - 1; i >= 0; i -= 1) {
|
||||
const entry = arr[i] || {};
|
||||
if (entry.book === book && entry.stats?.sourceIndex === '40krpgtools') {
|
||||
arr.splice(i, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
const existing = new Set(arr.map(entry => norm(entry.bestiaryName || entry.name)).filter(Boolean));
|
||||
const rows = index.entries.filter(row => row.book === book && !existing.has(norm(row.name)));
|
||||
|
||||
const pageCache = new Map();
|
||||
const parsedByPage = new Map();
|
||||
const added = [];
|
||||
const skipped = [];
|
||||
|
||||
for (const row of rows) {
|
||||
if (limit && added.length >= limit) break;
|
||||
if (!pageCache.has(row.page)) {
|
||||
const text = textForPage(book, row.page);
|
||||
pageCache.set(row.page, text);
|
||||
parsedByPage.set(row.page, parseProfilesFromPage(text));
|
||||
}
|
||||
const parsed = parsedByPage.get(row.page);
|
||||
const hit = parsed.find(candidate => {
|
||||
const a = norm(candidate.name);
|
||||
const b = norm(row.name);
|
||||
return a === b || a.includes(b) || b.includes(a);
|
||||
});
|
||||
if (!hit) {
|
||||
skipped.push(`${row.name} (${book} ${row.page})`);
|
||||
continue;
|
||||
}
|
||||
arr.push(toEntry(hit, row, book));
|
||||
existing.add(norm(row.name));
|
||||
added.push(`${row.name} (${book} ${row.page})`);
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
if (!Array.isArray(raw)) raw.count = arr.length;
|
||||
const serialized = JSON.stringify(raw, null, 2);
|
||||
for (const file of [PUBLIC_PATH, DB_PATH]) {
|
||||
if (fs.existsSync(file)) fs.writeFileSync(file + '.pre-hydrate.' + Date.now() + '.json', fs.readFileSync(file));
|
||||
fs.writeFileSync(file, serialized);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`${dryRun ? 'Would add' : 'Added'} ${added.length} entries from ${book}`);
|
||||
added.forEach(name => console.log(` + ${name}`));
|
||||
console.log(`Skipped ${skipped.length} rows without a parsed profile match`);
|
||||
skipped.slice(0, 20).forEach(name => console.log(` - ${name}`));
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user