feat: Add scripts for upserting and validating bestiary data
- Implemented `upsert-allewis-from-5001.js` to fetch and update Allewis data from API. - Created `upsert-missing-movement-from-5001.js` to fill missing movement and wounds data from API. - Developed `upsert-missing-movement-from-pdfs.js` to extract movement and wounds from PDF text. - Added `upsert-missing-profiles-from-pdfs.js` to fill missing profiles from PDF data. - Introduced `upsert-missing-wounds-from-pdfs.js` to update wounds data from PDF sources. - Created `validate-bestiary.js` to validate the structure of the bestiary JSON. - Updated `BestiaryTab.jsx` component to display bestiary data with improved structure and search functionality. - Added tests for `BestiaryTab` to ensure proper rendering and functionality.
This commit is contained in:
59
scripts/choose-and-parse-alexei-pdfjs.js
Normal file
59
scripts/choose-and-parse-alexei-pdfjs.js
Normal file
@@ -0,0 +1,59 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const dataPath = path.resolve(__dirname, '../database/alexei-pdfjs.json');
|
||||
if (!fs.existsSync(dataPath)) { console.error('missing file:', dataPath); process.exit(2); }
|
||||
const j = JSON.parse(fs.readFileSync(dataPath,'utf8'));
|
||||
const findings = j.findings || [];
|
||||
function scoreFinding(f){
|
||||
const s = f.text.toLowerCase();
|
||||
let score = 0;
|
||||
if (s.includes('alexei')) score += 5;
|
||||
if (s.includes('drahj')) score += 5;
|
||||
if (/movement:\s*\d/.test(s)) score += 4;
|
||||
if (/wounds:\s*\d/.test(s)) score += 4;
|
||||
if (/profile\s*wS/i.test(s)) score += 3;
|
||||
if (/wS\s+BS\s+S\s+T\s+Ag\s+Int/i.test(s)) score += 2;
|
||||
if (/\bskills:\b/.test(s)) score += 1;
|
||||
if (/\bweapons:\b/.test(s)) score += 1;
|
||||
return score;
|
||||
}
|
||||
function findBest(){
|
||||
let best = null;
|
||||
for (const f of findings){
|
||||
const sc = scoreFinding(f);
|
||||
f._score = sc;
|
||||
if (!best || sc > best._score) best = f;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
function extractFromText(text){
|
||||
const out = {};
|
||||
const m = text.match(/Movement:\s*([0-9\/]+(?:\/[0-9]+)*)/i);
|
||||
out.movement = m ? m[1].trim() : null;
|
||||
const w = text.match(/Wounds:\s*(\d{1,3})/i);
|
||||
out.wounds = w ? parseInt(w[1],10) : null;
|
||||
// profile: try label 'Profile' line, or a row of numbers after 'Profile' or after name
|
||||
const profLabel = text.match(/Profile[^\n]*\n?([\s\S]{0,200})/i);
|
||||
if (profLabel && profLabel[1]){
|
||||
const nums = profLabel[1].replace(/[()]/g,'').match(/(\d{1,2})(?:[^\d]+(\d{1,2})){8}/);
|
||||
if (nums){
|
||||
// fallback, simpler: find first 9 numbers in nearby text
|
||||
}
|
||||
}
|
||||
// simpler: find first sequence of 9 numbers in the snippet
|
||||
const seq = text.match(/(\d{1,2})(?:\s+\(?\d{1,2}\)?){8}/);
|
||||
if (seq){
|
||||
const nums = seq[0].replace(/[()]/g,'').trim().split(/\s+/).map(n=>parseInt(n,10));
|
||||
if (nums.length===9){
|
||||
const keys = ['ws','bs','s','t','ag','int','per','wp','fel'];
|
||||
out.profile = {};
|
||||
keys.forEach((k,i)=>out.profile[k]=nums[i]);
|
||||
}
|
||||
} else out.profile = null;
|
||||
return out;
|
||||
}
|
||||
const best = findBest();
|
||||
if (!best){ console.error('no candidate'); process.exit(1); }
|
||||
const extracted = extractFromText(best.text);
|
||||
const result = { pdf: best.pdf || 'unknown', page: best.page, score: best._score, extracted, snippet: best.text.slice(0,1000).replace(/\n+/g,' ') };
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
106
scripts/clean-enemies-raw.js
Normal file
106
scripts/clean-enemies-raw.js
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env node
|
||||
// Clean and dedupe enemies-raw.json into a compact structured file
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const IN_FILE = path.join(__dirname, '..', 'database', 'enemies-raw.json');
|
||||
const OUT_FILE = path.join(__dirname, '..', 'database', 'enemies-clean.json');
|
||||
const OUT_SAMPLE = path.join(__dirname, '..', 'database', 'enemies-clean-sample.json');
|
||||
|
||||
function normalizeName(s) {
|
||||
if (!s) return '';
|
||||
let t = s.replace(/\(.*?\)/g, ''); // remove parentheses
|
||||
t = t.replace(/[^\w\s\-\:']/g, ' '); // keep words, spaces, dashes, colons, apostrophes
|
||||
t = t.replace(/\bthe\b\s*/i, '');
|
||||
t = t.replace(/[\-:]+/g, ' ');
|
||||
t = t.replace(/\s+/g, ' ').trim();
|
||||
t = t.toLowerCase();
|
||||
return t;
|
||||
}
|
||||
|
||||
function chooseRepresentative(namesMap) {
|
||||
// namesMap: Map originalName -> count
|
||||
let best = null;
|
||||
let bestCount = 0;
|
||||
for (const [name, count] of namesMap.entries()) {
|
||||
if (count > bestCount) { best = name; bestCount = count; }
|
||||
}
|
||||
if (best) return best;
|
||||
return Array.from(namesMap.keys())[0] || '';
|
||||
}
|
||||
|
||||
function mergeStats(dest, src) {
|
||||
if (!src || typeof src !== 'object') return;
|
||||
for (const k of ['wounds','toughness','ap','page']) {
|
||||
const v = src[k];
|
||||
if (typeof v === 'number') {
|
||||
if (typeof dest[k] !== 'number') dest[k] = v;
|
||||
else dest[k] = Math.max(dest[k], v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function topUnique(arr, n=3) {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
for (const s of arr) {
|
||||
if (!s) continue;
|
||||
const t = s.trim();
|
||||
if (!t) continue;
|
||||
if (seen.has(t)) continue;
|
||||
seen.add(t);
|
||||
out.push(t);
|
||||
if (out.length >= n) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (!fs.existsSync(IN_FILE)) { console.error('Input file not found:', IN_FILE); process.exit(1); }
|
||||
const raw = JSON.parse(fs.readFileSync(IN_FILE, 'utf8'));
|
||||
const groups = new Map();
|
||||
|
||||
for (const entry of raw) {
|
||||
const orig = (entry.name || '').trim();
|
||||
const key = normalizeName(orig || (entry.snippet||'').split('\n')[0]);
|
||||
if (!key) continue;
|
||||
if (!groups.has(key)) {
|
||||
groups.set(key, {
|
||||
key,
|
||||
count: 0,
|
||||
names: new Map(),
|
||||
sources: new Set(),
|
||||
snippets: [],
|
||||
stats: {},
|
||||
});
|
||||
}
|
||||
const g = groups.get(key);
|
||||
g.count += 1;
|
||||
g.names.set(orig, (g.names.get(orig)||0) + 1);
|
||||
if (entry.source) g.sources.add(entry.source);
|
||||
if (entry.snippet) g.snippets.push(entry.snippet);
|
||||
if (entry.stats) mergeStats(g.stats, entry.stats);
|
||||
}
|
||||
|
||||
const output = [];
|
||||
for (const g of groups.values()) {
|
||||
const rep = chooseRepresentative(g.names);
|
||||
output.push({
|
||||
id: g.key,
|
||||
name: rep,
|
||||
count: g.count,
|
||||
sources: Array.from(g.sources),
|
||||
snippets: topUnique(g.snippets, 3),
|
||||
stats: g.stats
|
||||
});
|
||||
}
|
||||
|
||||
// sort by count desc
|
||||
output.sort((a,b) => b.count - a.count);
|
||||
fs.writeFileSync(OUT_FILE, JSON.stringify(output, null, 2));
|
||||
fs.writeFileSync(OUT_SAMPLE, JSON.stringify(output.slice(0, 200), null, 2));
|
||||
console.log('Wrote', OUT_FILE, 'with', output.length, 'unique entries');
|
||||
}
|
||||
|
||||
if (require.main === module) main();
|
||||
40
scripts/copy-bestiary-live.js
Normal file
40
scripts/copy-bestiary-live.js
Normal file
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Simple script to copy bestiary from database to public for live updates
|
||||
function copyBestiaryToPublic() {
|
||||
const dbPath = path.join(__dirname, '../database/deathwatch-bestiary-extracted.json');
|
||||
const publicPath = path.join(__dirname, '../public/deathwatch-bestiary-extracted.json');
|
||||
const buildPath = path.join(__dirname, '../build/deathwatch-bestiary-extracted.json');
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(dbPath)) {
|
||||
console.error('❌ Database bestiary file not found:', dbPath);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Copy to public
|
||||
fs.copyFileSync(dbPath, publicPath);
|
||||
console.log('✅ Copied to public:', publicPath);
|
||||
|
||||
// Copy to build if it exists
|
||||
if (fs.existsSync(path.dirname(buildPath))) {
|
||||
fs.copyFileSync(dbPath, buildPath);
|
||||
console.log('✅ Copied to build:', buildPath);
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('❌ Error copying bestiary files:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
copyBestiaryToPublic();
|
||||
}
|
||||
|
||||
module.exports = { copyBestiaryToPublic };
|
||||
27
scripts/copy-bestiary-to-public.js
Normal file
27
scripts/copy-bestiary-to-public.js
Normal file
@@ -0,0 +1,27 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const repoRoot = path.resolve(__dirname, '..');
|
||||
const src = path.join(repoRoot, 'database', 'deathwatch-bestiary-extracted.json');
|
||||
const dst = path.join(repoRoot, 'public', 'deathwatch-bestiary-extracted.json');
|
||||
|
||||
if (!fs.existsSync(src)) {
|
||||
console.error('Source not found:', src);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const data = JSON.parse(fs.readFileSync(src, 'utf8'));
|
||||
if (!Array.isArray(data.results)) {
|
||||
console.error('Source appears invalid, missing results array');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// backup dst if exists
|
||||
if (fs.existsSync(dst)) {
|
||||
const bak = dst + `.backup.${Date.now()}.json`;
|
||||
fs.copyFileSync(dst, bak);
|
||||
console.log('Backed up existing public file to', bak);
|
||||
}
|
||||
|
||||
fs.writeFileSync(dst, JSON.stringify(data, null, 2), 'utf8');
|
||||
console.log('Copied normalized bestiary to', dst);
|
||||
24
scripts/dump-pdf-pages.js
Normal file
24
scripts/dump-pdf-pages.js
Normal file
@@ -0,0 +1,24 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const pdfjs = require('pdfjs-dist/legacy/build/pdf.js');
|
||||
(async ()=>{
|
||||
const pdfPath = path.resolve(__dirname, '../data/Deathwatch - The Emperor Protects.pdf');
|
||||
if (!fs.existsSync(pdfPath)) { console.error('PDF not found at', pdfPath); process.exit(2); }
|
||||
const data = new Uint8Array(fs.readFileSync(pdfPath));
|
||||
const loadingTask = pdfjs.getDocument({data});
|
||||
const doc = await loadingTask.promise;
|
||||
const pages = [88,89];
|
||||
for (const pnum of pages){
|
||||
try{
|
||||
const page = await doc.getPage(pnum);
|
||||
const content = await page.getTextContent();
|
||||
const text = content.items.map(i=>i.str).join(' ');
|
||||
console.log('----- PAGE', pnum, 'START -----');
|
||||
console.log(text);
|
||||
console.log('----- PAGE', pnum, 'END -----\n');
|
||||
}catch(err){
|
||||
console.error('error reading page', pnum, err.message);
|
||||
}
|
||||
}
|
||||
process.exit(0);
|
||||
})();
|
||||
109
scripts/extract-alexei-pdfjs.js
Normal file
109
scripts/extract-alexei-pdfjs.js
Normal file
@@ -0,0 +1,109 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const pdfjsLib = require('pdfjs-dist/legacy/build/pdf.js');
|
||||
|
||||
const DATA_DIR = path.resolve(__dirname, '..', 'data');
|
||||
const targetPdf = 'Deathwatch - The Emperor Protects.pdf';
|
||||
const pdfPath = path.join(DATA_DIR, targetPdf);
|
||||
const outPath = path.resolve(__dirname, '..', 'database', 'alexei-pdfjs.json');
|
||||
const nameRegex = /alexei\s+drahj/i;
|
||||
|
||||
function extractStatsFromText(text){
|
||||
const out={profile:null,movement:null,wounds:null,toughness:null,skills:null,talents:null,traits:null,armour:null,weapons:null,gear:null,snippet:null};
|
||||
if(!text) return out;
|
||||
const t = text.replace(/\r/g,'\n');
|
||||
const lines = t.split('\n').map(l=>l.trim()).filter(Boolean);
|
||||
const joined = lines.join('\n');
|
||||
|
||||
let prof = null;
|
||||
const profHeader = joined.match(/ws\W*bs\W*s\W*t\W*ag\W*int\W*per\W*wp\W*fel/i);
|
||||
if(profHeader){
|
||||
const after = joined.slice(profHeader.index + profHeader[0].length, profHeader.index + profHeader[0].length + 400);
|
||||
const nums = (after.match(/\d{1,3}/g) || []).slice(0,9);
|
||||
if(nums.length>=9) prof = nums.map(n=>parseInt(n,10));
|
||||
} else {
|
||||
const pMatch = joined.match(/profile[\s\S]{0,300}?(\d{1,3}[\s\S]*?\d{1,3})/i);
|
||||
if(pMatch){ const nums = (pMatch[0].match(/\d{1,3}/g)||[]).slice(0,9); if(nums.length>=9) prof = nums.map(n=>parseInt(n,10)); }
|
||||
}
|
||||
if(prof) out.profile = { ws:prof[0], bs:prof[1], s:prof[2], t:prof[3], ag:prof[4], int:prof[5], per:prof[6], wp:prof[7], fel:prof[8] };
|
||||
|
||||
const mv = joined.match(/movement[:\s]*([0-9]+\/[0-9]+\/[0-9]+\/[0-9]+)/i) || joined.match(/\b(\d+\/\d+\/\d+\/\d+)\b/);
|
||||
if(mv) out.movement = mv[1] ? mv[1].trim() : mv[0].trim();
|
||||
|
||||
const wMatch = joined.match(/wounds?[:\s]*([0-9]{1,3})/i);
|
||||
if(wMatch) out.wounds = parseInt(wMatch[1],10);
|
||||
|
||||
const toughMatch = joined.match(/toughness[:\s]*([0-9]{1,3})/i);
|
||||
if(toughMatch) out.toughness = parseInt(toughMatch[1],10);
|
||||
|
||||
function captureSectionByHeaders(text, label){
|
||||
const re = new RegExp(label+':?\s*([\s\S]*?)(?=\n(?:Skills|Talents|Traits|Armou?r|Weapons|Gear|$):?)','i');
|
||||
const m = text.match(re);
|
||||
if(!m) return null;
|
||||
return m[1].replace(/\n+/g,' ').replace(/\s+/g,' ').trim();
|
||||
}
|
||||
|
||||
out.skills = captureSectionByHeaders(joined, 'Skills');
|
||||
out.talents = captureSectionByHeaders(joined, 'Talents');
|
||||
out.traits = captureSectionByHeaders(joined, 'Traits');
|
||||
out.armour = captureSectionByHeaders(joined, 'Armour') || captureSectionByHeaders(joined, 'Armor');
|
||||
out.weapons = captureSectionByHeaders(joined, 'Weapons');
|
||||
out.gear = captureSectionByHeaders(joined, 'Gear');
|
||||
|
||||
out.snippet = lines.slice(0,8).join(' ').replace(/\s+/g,' ').trim().slice(0,800);
|
||||
return out;
|
||||
}
|
||||
|
||||
(async function main(){
|
||||
if(!fs.existsSync(pdfPath)){ console.error('PDF not found:', pdfPath); process.exit(1); }
|
||||
const data = new Uint8Array(fs.readFileSync(pdfPath));
|
||||
const loadingTask = pdfjsLib.getDocument({data});
|
||||
const doc = await loadingTask.promise;
|
||||
const np = doc.numPages;
|
||||
const findings = [];
|
||||
|
||||
for(let i=1;i<=np;i++){
|
||||
try{
|
||||
const page = await doc.getPage(i);
|
||||
const content = await page.getTextContent();
|
||||
const pageText = content.items.map(it=>it.str).join(' ');
|
||||
if(nameRegex.test(pageText)){
|
||||
findings.push({page:i,text:pageText});
|
||||
}
|
||||
}catch(err){ console.error('page err', i, err && err.message); }
|
||||
}
|
||||
|
||||
if(findings.length===0){
|
||||
// fallback: search whole document text
|
||||
const all = [];
|
||||
for(let i=1;i<=np;i++){
|
||||
const page = await doc.getPage(i);
|
||||
const content = await page.getTextContent();
|
||||
const pageText = content.items.map(it=>it.str).join(' ');
|
||||
all.push(pageText);
|
||||
}
|
||||
for(let i=0;i<all.length;i++) if(nameRegex.test(all[i])) findings.push({page:i+1,text:all[i]});
|
||||
}
|
||||
|
||||
// For each finding, combine -2..+2 pages and extract stats
|
||||
const results = [];
|
||||
for(const f of findings){
|
||||
const idx = f.page;
|
||||
const start = Math.max(1, idx-2);
|
||||
const end = Math.min(np, idx+2);
|
||||
const parts = [];
|
||||
for(let p = start; p<=end; p++){
|
||||
const page = await doc.getPage(p);
|
||||
const c = await page.getTextContent();
|
||||
parts.push(c.items.map(it=>it.str).join(' '));
|
||||
}
|
||||
const combined = parts.join('\n\f\n');
|
||||
const stats = extractStatsFromText(combined);
|
||||
stats.chosenPage = idx;
|
||||
stats.chosenOffset = 0; // pdfjs gives true page numbers; offset left as 0
|
||||
results.push({pdf: targetPdf, foundPage: idx, range: [start,end], stats});
|
||||
}
|
||||
|
||||
fs.writeFileSync(outPath, JSON.stringify({generatedAt: new Date().toISOString(), findings, results}, null, 2));
|
||||
console.log('Wrote', outPath, 'findings:', findings.length);
|
||||
})();
|
||||
96
scripts/extract-enemies-from-pdfs.js
Normal file
96
scripts/extract-enemies-from-pdfs.js
Normal file
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env node
|
||||
// Extract candidate enemy names and nearby stat snippets from PDFs in /data
|
||||
// Usage: node scripts/extract-enemies-from-pdfs.js
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const pdf = require('pdf-parse');
|
||||
|
||||
const DATA_DIR = path.join(__dirname, '..', 'data');
|
||||
const OUT_DIR = path.join(__dirname, '..', 'database');
|
||||
const OUT_FILE = path.join(OUT_DIR, 'enemies-raw.json');
|
||||
|
||||
function safeMkdir(dir) {
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
function guessNamesFromText(text) {
|
||||
// Split into candidate lines by newlines. Heuristic: lines with 2-5 Titlecase words and length < 90
|
||||
const lines = text.split(/\r?\n/).map(l => l.trim()).filter(Boolean);
|
||||
const candidates = new Set();
|
||||
for (const line of lines) {
|
||||
if (line.length < 6 || line.length > 120) continue;
|
||||
// Skip lines that look like table headers or page footers
|
||||
if (/^(Page|Table|Contents|Chapter)\b/i.test(line)) continue;
|
||||
// Count Titlecase words
|
||||
const words = line.split(/\s+/);
|
||||
const titleCount = words.filter(w => /^[A-Z0-9][a-zA-Z'\-\(\)]{1,}/.test(w)).length;
|
||||
if (titleCount >= 2 && titleCount <= 6) {
|
||||
// Avoid lines with many punctuation marks
|
||||
if ((line.match(/[\|\=\*]/g) || []).length > 0) continue;
|
||||
// Likely a name/title
|
||||
candidates.add(line.replace(/\s{2,}/g, ' '));
|
||||
}
|
||||
}
|
||||
return Array.from(candidates);
|
||||
}
|
||||
|
||||
function extractStatsFromSnippet(snippet) {
|
||||
const out = {};
|
||||
// Wounds
|
||||
const wounds = snippet.match(/Wounds[:\s]*([0-9]{1,3})/i) || snippet.match(/wounds[:\s]*([0-9]{1,3})/i);
|
||||
if (wounds) out.wounds = Number(wounds[1]);
|
||||
// Toughness / TB
|
||||
const tb = snippet.match(/\bToughness[:\s]*([0-9]{1,3})/i) || snippet.match(/\bTB[:\s]*([0-9]{1,3})/i);
|
||||
if (tb) out.toughness = Number(tb[1]);
|
||||
// Armour / AP
|
||||
const ap = snippet.match(/Armou?r(?:\s|:)?\s*AP[:\s]*([0-9]{1,3})/i) || snippet.match(/AP[:\s]*([0-9]{1,3})/i);
|
||||
if (ap) out.ap = Number(ap[1]);
|
||||
// Page number hints
|
||||
const page = snippet.match(/Page\s+No\.\s*([0-9]{1,4})/i) || snippet.match(/p(?:age)?\.?\s*([0-9]{1,4})/i);
|
||||
if (page) out.page = Number(page[1]);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function processPdf(filePath) {
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
try {
|
||||
const data = await pdf(buffer);
|
||||
const text = data.text || '';
|
||||
const names = guessNamesFromText(text);
|
||||
const entries = [];
|
||||
for (const name of names) {
|
||||
const idx = text.indexOf(name);
|
||||
if (idx === -1) continue;
|
||||
const start = Math.max(0, idx - 300);
|
||||
const end = Math.min(text.length, idx + name.length + 400);
|
||||
const snippet = text.substring(start, end).replace(/\s{2,}/g, ' ');
|
||||
const stats = extractStatsFromSnippet(snippet);
|
||||
entries.push({ name, source: path.basename(filePath), snippet: snippet.trim(), stats });
|
||||
}
|
||||
return entries;
|
||||
} catch (err) {
|
||||
console.error('Error parsing PDF', filePath, err.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
safeMkdir(OUT_DIR);
|
||||
if (!fs.existsSync(DATA_DIR)) {
|
||||
console.error('Data directory not found:', DATA_DIR);
|
||||
process.exit(1);
|
||||
}
|
||||
const files = fs.readdirSync(DATA_DIR).filter(f => f.toLowerCase().endsWith('.pdf'));
|
||||
const all = [];
|
||||
for (const f of files) {
|
||||
const p = path.join(DATA_DIR, f);
|
||||
console.log('Processing', f);
|
||||
const entries = await processPdf(p);
|
||||
all.push(...entries);
|
||||
}
|
||||
fs.writeFileSync(OUT_FILE, JSON.stringify(all, null, 2));
|
||||
console.log('Wrote', OUT_FILE, 'with', all.length, 'candidate entries');
|
||||
}
|
||||
|
||||
if (require.main === module) main().catch(err => { console.error(err); process.exit(1); });
|
||||
73
scripts/extract-exact-alexei-from-pdfjs.js
Normal file
73
scripts/extract-exact-alexei-from-pdfjs.js
Normal file
@@ -0,0 +1,73 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const dataPath = path.resolve(__dirname, '../database/alexei-pdfjs.json');
|
||||
if (!fs.existsSync(dataPath)) { console.error('missing file:', dataPath); process.exit(2); }
|
||||
const j = JSON.parse(fs.readFileSync(dataPath,'utf8'));
|
||||
const findings = j.findings || [];
|
||||
// find any snippet containing 'alexei' or 'drahj'
|
||||
const hits = findings.filter(f => /alexei|drahj/i.test(f.text));
|
||||
if (hits.length===0) { console.error('no hits'); process.exit(1); }
|
||||
// Prefer hits where 'Profile' or 'Movement' occurs nearby
|
||||
function score(f){
|
||||
let s=0; const t = f.text.toLowerCase();
|
||||
if (t.includes('movement:')) s+=4;
|
||||
if (t.includes('wounds:')) s+=4;
|
||||
if (t.includes('profile')) s+=2;
|
||||
if (t.includes('talents')) s+=1;
|
||||
if (t.includes('weapons')) s+=1;
|
||||
if (/alexei\s+drahj/.test(t)) s+=5;
|
||||
return s;
|
||||
}
|
||||
let best = hits[0];
|
||||
for (const h of hits){ if (score(h) > score(best)) best = h; }
|
||||
const txt = best.text;
|
||||
// locate alexei token index
|
||||
let idx = txt.search(/alexei\s+drahj/i);
|
||||
if (idx<0) idx = txt.search(/alexei/i);
|
||||
const windowText = txt.substr(Math.max(0, idx-300), 1200);
|
||||
function extractProfile(s){
|
||||
// Handle optional leading parenthetical modifiers like "(10) (12) (8)" followed by the main 9-value profile row.
|
||||
const modifiersMatch = s.match(/^(?:[^\n]{0,80})?\(?\s*(\d{1,2})\s*\)?(?:\s*\(?\s*(\d{1,2})\s*\)\s*)?(?:\(?\s*(\d{1,2})\s*\)\s*)?/m);
|
||||
let modifiers = null;
|
||||
if (modifiersMatch && (modifiersMatch[1] || modifiersMatch[2] || modifiersMatch[3])) {
|
||||
modifiers = [modifiersMatch[1], modifiersMatch[2], modifiersMatch[3]].filter(Boolean).map(n=>parseInt(n,10));
|
||||
}
|
||||
// Find the first 9-number sequence that looks like the canonical profile (allow some separators)
|
||||
const seqAll = Array.from(s.matchAll(/(?:\b\d{1,2}\b(?:[^\d\n]{1,6}\d{1,2}\b){8})/gm));
|
||||
if (!seqAll || seqAll.length===0) return null;
|
||||
// Prefer a sequence that occurs after the modifiers (if modifiers present), otherwise take the last sequence
|
||||
let chosenSeq = null;
|
||||
if (modifiers) {
|
||||
for (const m of seqAll) {
|
||||
if (m.index > (modifiersMatch.index || 0)) { chosenSeq = m[0]; break; }
|
||||
}
|
||||
}
|
||||
if (!chosenSeq) chosenSeq = seqAll[seqAll.length-1][0];
|
||||
const nums = chosenSeq.replace(/[^0-9\s]/g,' ').trim().split(/\s+/).map(n=>parseInt(n,10));
|
||||
if (nums.length<9) return null;
|
||||
const keys = ['ws','bs','s','t','ag','int','per','wp','fel'];
|
||||
const obj = {};
|
||||
keys.forEach((k,i)=>obj[k]=nums[i]);
|
||||
// if modifiers were found, attach them separately
|
||||
if (modifiers) obj.modifiers = modifiers;
|
||||
return obj;
|
||||
}
|
||||
function extractMovement(s){ const m = s.match(/Movement:\s*([0-9\/]+(?:\/[0-9]+)*)/i); return m?m[1].trim():null; }
|
||||
function extractWounds(s){ const m = s.match(/Wounds:\s*(\d{1,3})/i); return m?parseInt(m[1],10):null; }
|
||||
function extractWeapons(s){ // capture a weapons line block following 'Weapons:' up to two lines
|
||||
const m = s.match(/Weapons:\s*([\s\S]{0,400})/i);
|
||||
if (!m) return null;
|
||||
// stop at 'Gear' or double newline
|
||||
let blk = m[1].split(/\n\s*\n/)[0];
|
||||
blk = blk.split(/Gear:|Armour:|Talents:|Traits:/i)[0];
|
||||
return blk.replace(/\n+/g,' ').trim();
|
||||
}
|
||||
const profile = extractProfile(windowText);
|
||||
const movement = extractMovement(windowText);
|
||||
const wounds = extractWounds(windowText);
|
||||
const weapons = extractWeapons(windowText);
|
||||
const out = {
|
||||
source: { pdf: best.pdf || null, page: best.page, range: best.range || null },
|
||||
profile, movement, wounds, weapons, snippet: windowText.replace(/\n+/g,' ').trim().slice(0,2000)
|
||||
};
|
||||
console.log(JSON.stringify(out, null, 2));
|
||||
41
scripts/extract-pages-for-name.js
Normal file
41
scripts/extract-pages-for-name.js
Normal file
@@ -0,0 +1,41 @@
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const pdf = require('pdf-parse');
|
||||
|
||||
async function extract(nameRegex) {
|
||||
const dataDir = path.resolve(__dirname, '../data');
|
||||
const outFile = path.resolve(__dirname, '../database/alexei-pages.json');
|
||||
const files = await fs.readdir(dataDir);
|
||||
const pdfs = files.filter(f => f.toLowerCase().endsWith('.pdf'));
|
||||
|
||||
const results = [];
|
||||
for (const pdfName of pdfs) {
|
||||
const filePath = path.join(dataDir, pdfName);
|
||||
try {
|
||||
const dataBuffer = await fs.readFile(filePath);
|
||||
const parsed = await pdf(dataBuffer);
|
||||
const text = parsed.text || '';
|
||||
const pages = text.split('\f');
|
||||
for (let idx = 0; idx < pages.length; idx++) {
|
||||
const pageText = pages[idx];
|
||||
if (nameRegex.test(pageText)) {
|
||||
// Include surrounding pages for context
|
||||
const context = [];
|
||||
for (let j = Math.max(0, idx-2); j <= Math.min(pages.length-1, idx+2); j++) {
|
||||
context.push({ pageIndex: j, text: pages[j].trim() });
|
||||
}
|
||||
results.push({ pdf: pdfName, matchPageIndex: idx, context });
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('failed', pdfName, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
const out = { generatedAt: new Date().toISOString(), matches: results };
|
||||
await fs.writeFile(outFile, JSON.stringify(out, null, 2));
|
||||
console.log('Wrote', outFile, 'matches:', results.length);
|
||||
}
|
||||
|
||||
const nameRegex = /alexei\s+drahj/i;
|
||||
extract(nameRegex).catch(err => { console.error(err); process.exit(2); });
|
||||
69
scripts/fetch-enemies.js
Normal file
69
scripts/fetch-enemies.js
Normal file
@@ -0,0 +1,69 @@
|
||||
// Some axios/undici versions expect a global `File` constructor to exist
|
||||
// in certain Node environments. Polyfill a minimal noop File if missing
|
||||
// so the script can run in CI/dev Node without changing dependencies.
|
||||
if (typeof File === 'undefined') global.File = class File {};
|
||||
const axios = require('axios');
|
||||
const cheerio = require('cheerio');
|
||||
const { URL } = require('url');
|
||||
|
||||
async function fetchEnemies(pageUrl = 'http://www.40krpgtools.com/library/bestiary/') {
|
||||
try {
|
||||
const res = await axios.get(pageUrl);
|
||||
const html = res.data;
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const out = [];
|
||||
// Try to select rows from the bestiary table body (server-side rendered)
|
||||
$('#bestiaryTable tbody tr').each((i, row) => {
|
||||
const cols = $(row).find('td');
|
||||
if (cols.length < 6) return; // skip malformed rows
|
||||
|
||||
const nameCell = $(cols[0]);
|
||||
const name = nameCell.text().trim();
|
||||
const rel = nameCell.find('a').attr('href') || '';
|
||||
const fullUrl = rel ? (new URL(rel, pageUrl)).toString() : '';
|
||||
const type = $(cols[1]).text().trim();
|
||||
const affiliation = $(cols[2]).text().trim();
|
||||
const setting = $(cols[3]).text().trim();
|
||||
const book = $(cols[4]).text().trim();
|
||||
const pageNo = $(cols[5]).text().trim();
|
||||
|
||||
out.push({ name, type, affiliation, setting, book, pageNo, url: fullUrl });
|
||||
});
|
||||
|
||||
// Fallback: some pages render the table client-side. If we found nothing,
|
||||
// try to collect library links (e.g. /library/deathwatch/...) and turn
|
||||
// them into a basic list of entries (name + url). This provides useful
|
||||
// results even when the full table is not present in the HTML.
|
||||
if (out.length === 0) {
|
||||
const seen = new Set();
|
||||
$('a[href^="/library/"]').each((i, a) => {
|
||||
const href = $(a).attr('href')
|
||||
const text = $(a).text().trim()
|
||||
if (!href || !text) return
|
||||
// we want links that look like entity pages (multiple path segments)
|
||||
const parts = href.replace(/^\//, '').split('/');
|
||||
if (parts.length < 3) return // skip category index links
|
||||
const fullUrl = (new URL(href, pageUrl)).toString()
|
||||
const key = fullUrl
|
||||
if (seen.has(key)) return
|
||||
seen.add(key)
|
||||
out.push({ name: text, url: fullUrl })
|
||||
})
|
||||
}
|
||||
|
||||
return out;
|
||||
} catch (err) {
|
||||
console.error('Error fetching/parsing enemies page:', err && err.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
const page = process.argv[2] || 'http://www.40krpgtools.com/library/bestiary/';
|
||||
fetchEnemies(page)
|
||||
.then(list => console.log(JSON.stringify(list, null, 2)))
|
||||
.catch(err => { console.error(err); process.exit(1) });
|
||||
}
|
||||
|
||||
module.exports = { fetchEnemies };
|
||||
0
scripts/fill-bestiary-placeholders.js
Normal file
0
scripts/fill-bestiary-placeholders.js
Normal file
187
scripts/harvest-bestiary-from-pdfs.js
Normal file
187
scripts/harvest-bestiary-from-pdfs.js
Normal file
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const https = require('https');
|
||||
const pdf = require('pdf-parse');
|
||||
|
||||
const BESTIARY_URL = 'https://www.40krpgtools.com/bestiary/';
|
||||
const DATA_DIR = path.resolve(__dirname, '..', 'data');
|
||||
const OUT_PATH = path.resolve(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json');
|
||||
|
||||
function stripTags(s){return s.replace(/<[^>]+>/g,' ').replace(/\s+/g,' ').trim();}
|
||||
function normalize(s){return (s||'').toString().toLowerCase().replace(/[\u2018\u2019\u201c\u201d]/g,'').replace(/[^a-z0-9 ]+/g,' ').replace(/\s+/g,' ').trim();}
|
||||
function levenshtein(a,b){a=a||'';b=b||'';const m=a.length,n=b.length; if(!m) return n; if(!n) return m; const dp=Array.from({length:m+1},()=>Array(n+1).fill(0)); for(let i=0;i<=m;i++)dp[i][0]=i; for(let j=0;j<=n;j++)dp[0][j]=j; for(let i=1;i<=m;i++){ for(let j=1;j<=n;j++){ const c=a[i-1]===b[j-1]?0:1; dp[i][j]=Math.min(dp[i-1][j]+1, dp[i][j-1]+1, dp[i-1][j-1]+c); } } return dp[m][n]; }
|
||||
|
||||
function fetch(url, timeout=20000){ return new Promise((resolve,reject)=>{ const req=https.get(url,{timeout},res=>{ if(res.statusCode&&res.statusCode>=400) return reject(new Error('HTTP '+res.statusCode)); let s=''; res.setEncoding('utf8'); res.on('data',d=>s+=d); res.on('end',()=>resolve(s)); }); req.on('error',reject); req.on('timeout',()=>req.destroy(new Error('timeout'))); }); }
|
||||
|
||||
function parseBestiary(html){ const rows=[]; const trRe=/<tr[^>]*>([\s\S]*?)<\/tr>/gi; let m; while((m=trRe.exec(html))){ const tr=m[1]; const tdRe=/<td[^>]*>([\s\S]*?)<\/td>/gi; let mm; const cols=[]; while((mm=tdRe.exec(tr))){ cols.push(stripTags(mm[1])); } if(cols.length>=1){ const name=cols[0]||''; const page=cols[cols.length-1]||''; const book=cols[cols.length-2]||''; if(name) rows.push({name,book,page}); } } if(rows.length===0){ const lines=html.split('\n'); for(const line of lines){ if(/\|/.test(line)){ const parts=line.replace(/<[^>]+>/g,'').split('|').map(p=>p.trim()).filter(Boolean); if(parts.length>=3) rows.push({name:parts[0], book:parts[parts.length-2], page:parts[parts.length-1]}); } } } return rows; }
|
||||
|
||||
function findLocalPdfForBook(bookName, localFiles){ const n=normalize(bookName); if(!n) return null; let best=null; for(const f of localFiles){ const base=f.replace(/\.pdf$/i,''); const cand=normalize(base); if(cand.includes(n) || n.includes(cand)) return f; const d=levenshtein(n,cand); if(!best||d<best.d) best={d,f}; } return best?best.f:null; }
|
||||
|
||||
function extractStatsFromText(text){
|
||||
const out={
|
||||
profile: null, // {ws,bs,s,t,ag,int,per,wp,fel}
|
||||
movement: null,
|
||||
wounds: null,
|
||||
toughness: null,
|
||||
skills: null,
|
||||
talents: null,
|
||||
traits: null,
|
||||
armour: null,
|
||||
weapons: null,
|
||||
gear: null,
|
||||
snippet: null
|
||||
};
|
||||
if(!text) return out;
|
||||
const t = text.replace(/\r/g,'\n');
|
||||
// Normalize some line breaks for easier regexes
|
||||
const lines = t.split('\n').map(l=>l.trim()).filter(Boolean);
|
||||
const joined = lines.join('\n');
|
||||
|
||||
// 1) Profile: look for 'Profile' block containing labeled stats or inline header
|
||||
// Try labeled form: "WS BS S T Ag Int Per WP Fel" or similar
|
||||
const profileLabelsRe = /(profile[:\s\n]*)([\s\S]{0,200})/i;
|
||||
let prof = null;
|
||||
const profHeader = joined.match(/ws\W*bs\W*s\W*t\W*ag\W*int\W*per\W*wp\W*fel/i);
|
||||
if(profHeader){
|
||||
// find numbers following that header on same or next lines
|
||||
const after = joined.slice(profHeader.index + profHeader[0].length, profHeader.index + profHeader[0].length + 300);
|
||||
const nums = (after.match(/\d{1,3}/g) || []).slice(0,9);
|
||||
if(nums.length>=9){
|
||||
prof = nums.map(n=>parseInt(n,10));
|
||||
}
|
||||
} else {
|
||||
// fallback: look for 'Profile' block with 9 numbers nearby
|
||||
const pMatch = joined.match(/profile[\s\S]{0,200}?(\d{1,3}[\s\S]*?\d{1,3})/i);
|
||||
if(pMatch){ const nums = (pMatch[0].match(/\d{1,3}/g)||[]).slice(0,9); if(nums.length>=9) prof = nums.map(n=>parseInt(n,10)); }
|
||||
}
|
||||
if(prof){ out.profile = { ws:prof[0], bs:prof[1], s:prof[2], t:prof[3], ag:prof[4], int:prof[5], per:prof[6], wp:prof[7], fel:prof[8] }; }
|
||||
|
||||
// Movement: look for typical pattern like 4/8/12/24
|
||||
const mv = joined.match(/movement[:\s]*([0-9]+\/[0-9]+\/[0-9]+\/[0-9]+)/i) || joined.match(/\b(\d+\/\d+\/\d+\/\d+)\b/);
|
||||
if(mv) out.movement = mv[1] ? mv[1].trim() : mv[0].trim();
|
||||
|
||||
// Wounds: sometimes 'Wounds: 38' or 'Wounds 38' or 'Wounds: 38 (..)'
|
||||
const wMatch = joined.match(/wounds?[:\s]*([0-9]{1,3})/i);
|
||||
if(wMatch) out.wounds = parseInt(wMatch[1],10);
|
||||
|
||||
// Toughness
|
||||
const toughMatch = joined.match(/toughness[:\s]*([0-9]{1,3})/i);
|
||||
if(toughMatch) out.toughness = parseInt(toughMatch[1],10);
|
||||
|
||||
// Sections: capture via headers (Skills, Talents, Traits, Armour, Weapons, Gear)
|
||||
function captureSectionByHeaders(text, label){
|
||||
const re = new RegExp(label+':?\s*([\s\S]*?)(?=\n(?:Skills|Talents|Traits|Armou?r|Weapons|Gear|$):?)','i');
|
||||
const m = text.match(re);
|
||||
if(!m) return null;
|
||||
return m[1].replace(/\n+/g,' ').replace(/\s+/g,' ').trim();
|
||||
}
|
||||
|
||||
out.skills = captureSectionByHeaders(joined, 'Skills');
|
||||
out.talents = captureSectionByHeaders(joined, 'Talents');
|
||||
out.traits = captureSectionByHeaders(joined, 'Traits');
|
||||
out.armour = captureSectionByHeaders(joined, 'Armour') || captureSectionByHeaders(joined, 'Armor');
|
||||
out.weapons = captureSectionByHeaders(joined, 'Weapons');
|
||||
out.gear = captureSectionByHeaders(joined, 'Gear');
|
||||
|
||||
// final snippet: user doesn't want it in UI, but keep short cleaned snippet for logging
|
||||
out.snippet = lines.slice(0,6).join(' ').replace(/\s+/g,' ').trim().slice(0,800);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function extractPageText(pdfPath, pageNumber){ // pageNumber 1-based
|
||||
const data = fs.readFileSync(pdfPath);
|
||||
try{
|
||||
const parsed = await pdf(data);
|
||||
let pages = parsed.text.split('\f');
|
||||
if(pages.length && pages.length>=pageNumber){ return pages[pageNumber-1]; }
|
||||
// fallback: try approximate splitting by number of pages if info exists
|
||||
const np = parsed.numpages || pages.length;
|
||||
if(np>0 && pages.length!==np){ // attempt crude split by dividing text into np parts
|
||||
const allText = parsed.text.replace(/\s+/g,' ');
|
||||
const approxLen = Math.ceil(allText.length/np);
|
||||
const parts=[]; for(let i=0;i<np;i++){ parts.push(allText.slice(i*approxLen, (i+1)*approxLen)); }
|
||||
return parts[Math.max(0, Math.min(np-1, pageNumber-1))];
|
||||
}
|
||||
return parsed.text;
|
||||
}catch(err){ console.error('pdf parse error',pdfPath,err); return null; }
|
||||
}
|
||||
|
||||
async function main(){
|
||||
const html = await fetch(BESTIARY_URL);
|
||||
const rows = parseBestiary(html);
|
||||
console.log('Total bestiary rows:', rows.length);
|
||||
|
||||
// find local pdf files
|
||||
const localFiles = fs.existsSync(DATA_DIR) ? fs.readdirSync(DATA_DIR).filter(f=>/\.pdf$/i.test(f)) : [];
|
||||
console.log('Local PDF count:', localFiles.length);
|
||||
|
||||
// Filter rows to those referencing Deathwatch or matching local PDFs
|
||||
const filtered = rows.filter(r=>{
|
||||
const book = (r.book||'').toLowerCase();
|
||||
if(book.includes('deathwatch')) return true;
|
||||
for(const lf of localFiles){ if(book && normalize(book).includes(normalize(lf.replace(/\.pdf$/i,'')))) return true; }
|
||||
return false;
|
||||
});
|
||||
console.log('Filtered rows (likely Deathwatch):', filtered.length);
|
||||
|
||||
// If user expects 372, note difference
|
||||
if(filtered.length!==372) console.log('Warning: expected ~372, found',filtered.length);
|
||||
|
||||
const results=[];
|
||||
for(const r of filtered){
|
||||
const bookName = r.book || '';
|
||||
const pageNum = parseInt((r.page||'').replace(/[^0-9]/g,''),10) || null;
|
||||
const pdfFile = findLocalPdfForBook(bookName, localFiles);
|
||||
let pageText = null; let stats = null;
|
||||
if(pdfFile && pageNum){
|
||||
const pdfPath = path.join(DATA_DIR, pdfFile);
|
||||
// Try a small range of offsets to account for differences between printed page numbers
|
||||
// and PDF internal page indices (front matter, cover pages, etc.). We'll pick the
|
||||
// page with the best heuristic match to the entry name / stat tokens.
|
||||
const offsets = [-2, -1, 0, 1, 2];
|
||||
let best = {score:-1, text:null, offset:0};
|
||||
for(const off of offsets){
|
||||
const tryPage = pageNum + off;
|
||||
if(tryPage < 1) continue;
|
||||
const txt = await extractPageText(pdfPath, tryPage);
|
||||
if(!txt) continue;
|
||||
let score = 0;
|
||||
const lname = (r.name||'').toLowerCase();
|
||||
if(lname && txt.toLowerCase().includes(lname)) score += 10; // strong signal
|
||||
// presence of stat keywords
|
||||
const tokens = ['ws','bs','wounds','toughness','armour','movement','skills','talents','traits'];
|
||||
for(const t of tokens) if(txt.toLowerCase().includes(t)) score += 1;
|
||||
// small bonus for numeric tables
|
||||
if(/\b\d{1,2}\b/.test(txt)) score += 0.5;
|
||||
if(score > best.score){ best = {score, text:txt, offset:off}; }
|
||||
}
|
||||
if(best.score >= 0){
|
||||
pageText = best.text;
|
||||
stats = extractStatsFromText(pageText || '');
|
||||
if(best.offset !== 0){
|
||||
console.log(`Adjusted page for ${r.name} book='${bookName}' requested=${pageNum} -> used=${pageNum+best.offset} (offset=${best.offset})`);
|
||||
}
|
||||
// record offset used
|
||||
if(stats) stats.chosenOffset = best.offset;
|
||||
}
|
||||
} else if(pdfFile){
|
||||
const pdfPath = path.join(DATA_DIR, pdfFile);
|
||||
const parsed = await pdf(fs.readFileSync(pdfPath));
|
||||
pageText = (parsed.text||'').slice(0,400);
|
||||
stats = extractStatsFromText(pageText);
|
||||
} else {
|
||||
// try searching local PDFs for the name
|
||||
let found=null;
|
||||
for(const lf of localFiles){ const pdfPath = path.join(DATA_DIR, lf); const parsed = await pdf(fs.readFileSync(pdfPath)); if((parsed.text||'').toLowerCase().includes((r.name||'').toLowerCase())){ found={pdf:lf,text:(parsed.text||'').slice(0,400)}; break; } }
|
||||
if(found){ pageText = found.text; stats = extractStatsFromText(pageText); }
|
||||
}
|
||||
|
||||
results.push({bestiaryName: r.name, book: bookName, page: r.page, pdf: pdfFile, pageText: pageText? (pageText||'').slice(0,1000) : null, stats});
|
||||
}
|
||||
|
||||
fs.writeFileSync(OUT_PATH, JSON.stringify({generatedAt: new Date().toISOString(), count:results.length, results}, null, 2));
|
||||
console.log('Wrote', OUT_PATH);
|
||||
}
|
||||
|
||||
if(require.main===module) main().catch(err=>{ console.error(err); process.exit(1); });
|
||||
150
scripts/import-api-search-statblocks.js
Normal file
150
scripts/import-api-search-statblocks.js
Normal file
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env node
|
||||
// Import all statblocks from local search API (http://192.168.1.144:5001/api/search?q=)
|
||||
// - backs up existing bestiary
|
||||
// - upserts by name/page/source (normalizes names)
|
||||
// - prefers API movement/wounds/profile when present
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const http = require('http');
|
||||
|
||||
const API_URL = 'http://192.168.1.144:5001/api/search?q=';
|
||||
const BESTIARY_PATH = path.resolve(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json');
|
||||
const BACKUP_DIR = path.resolve(__dirname, '..', 'database', 'backups');
|
||||
|
||||
function fetchJson(url){
|
||||
return new Promise((resolve, reject) => {
|
||||
http.get(url, res => {
|
||||
let s = '';
|
||||
res.setEncoding('utf8');
|
||||
res.on('data', d => s += d);
|
||||
res.on('end', () => {
|
||||
try { resolve(JSON.parse(s)); }
|
||||
catch(e){ reject(e); }
|
||||
});
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeName(s){
|
||||
if(!s) return '';
|
||||
return s.toString().toLowerCase().replace(/[\u2018\u2019\u201c\u201d]/g,'')
|
||||
.replace(/[^a-z0-9]+/g,' ')
|
||||
.replace(/\s+/g,' ').trim();
|
||||
}
|
||||
|
||||
function ensureBackupDir(){
|
||||
if(!fs.existsSync(BACKUP_DIR)) fs.mkdirSync(BACKUP_DIR, {recursive:true});
|
||||
}
|
||||
|
||||
function backupFile(src){
|
||||
ensureBackupDir();
|
||||
const name = path.basename(src);
|
||||
const dest = path.join(BACKUP_DIR, `${name}.backup.${Date.now()}.json`);
|
||||
fs.copyFileSync(src, dest);
|
||||
return dest;
|
||||
}
|
||||
|
||||
function mergeObjects(existing, incoming){
|
||||
// shallow merge with preference to incoming for movement/wounds/profile when present
|
||||
const out = Object.assign({}, existing || {});
|
||||
for(const k of Object.keys(incoming || {})){
|
||||
const v = incoming[k];
|
||||
if(v===null || v===undefined) continue;
|
||||
if(typeof v === 'object' && !Array.isArray(v)){
|
||||
out[k] = mergeObjects(out[k] || {}, v);
|
||||
} else {
|
||||
out[k] = v;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function main(){
|
||||
console.log('Fetching statblocks from API:', API_URL);
|
||||
let data;
|
||||
try{ data = await fetchJson(API_URL); } catch(e){ console.error('Failed to fetch API:', e.message); process.exit(2); }
|
||||
const statblocks = data && (data.statblocks || data.results || []);
|
||||
if(!Array.isArray(statblocks)){
|
||||
console.error('API returned unexpected format'); process.exit(2);
|
||||
}
|
||||
console.log('Statblocks found on API:', statblocks.length);
|
||||
|
||||
// load or init bestiary
|
||||
let bestiaryObj = { generatedAt: new Date().toISOString(), count: 0, results: [] };
|
||||
if(fs.existsSync(BESTIARY_PATH)){
|
||||
try{ bestiaryObj = JSON.parse(fs.readFileSync(BESTIARY_PATH,'utf8')); }
|
||||
catch(e){ console.error('Failed to parse existing bestiary, aborting:', e.message); process.exit(2); }
|
||||
}
|
||||
if(!Array.isArray(bestiaryObj.results)) bestiaryObj.results = [];
|
||||
const results = bestiaryObj.results;
|
||||
|
||||
// backup before modifying
|
||||
if(fs.existsSync(BESTIARY_PATH)){
|
||||
const b = backupFile(BESTIARY_PATH);
|
||||
console.log('Backed up bestiary to', b);
|
||||
}
|
||||
|
||||
let added = 0, updated = 0;
|
||||
for(const sb of statblocks){
|
||||
const name = sb.name || sb.bestiaryName || sb.title || '';
|
||||
const norm = normalizeName(name);
|
||||
// find candidate by normalized name
|
||||
let idx = results.findIndex(r => normalizeName(r.bestiaryName || r.name || '') === norm);
|
||||
// fallback: match by page + source/book
|
||||
if(idx === -1 && sb.page){
|
||||
idx = results.findIndex(r => String(r.page) === String(sb.page) && ((r.book||'').toLowerCase().includes((sb.source||'').toLowerCase()) || (r.source||'').toLowerCase().includes((sb.source||'').toLowerCase())) );
|
||||
}
|
||||
|
||||
// build incoming entry in local schema
|
||||
const incoming = {};
|
||||
incoming.bestiaryName = name || undefined;
|
||||
if(sb.source) incoming.book = sb.source;
|
||||
if(sb.page) incoming.page = sb.page;
|
||||
if(sb.pdf) incoming.pdf = sb.pdf;
|
||||
// map stats
|
||||
incoming.stats = {};
|
||||
// profile
|
||||
if(sb.stats && sb.stats.profile) incoming.stats.profile = sb.stats.profile;
|
||||
else if(sb.profile) incoming.stats.profile = sb.profile;
|
||||
// movement/wounds from secondary_stats or stats
|
||||
incoming.stats.movement = (sb.stats && sb.stats.movement) || (sb.secondary_stats && sb.secondary_stats.movement) || sb.movement || undefined;
|
||||
incoming.stats.wounds = (sb.stats && sb.stats.wounds) || (sb.secondary_stats && sb.secondary_stats.wounds) || sb.wounds || undefined;
|
||||
// weapons/armour/skills/snippet
|
||||
if(sb.stats && sb.stats.weapons) incoming.stats.weapons = sb.stats.weapons;
|
||||
else if(sb.weapons) incoming.stats.weapons = sb.weapons;
|
||||
if(sb.stats && sb.stats.armour) incoming.stats.armour = sb.stats.armour;
|
||||
else if(sb.armour) incoming.stats.armour = sb.armour;
|
||||
if(sb.stats && sb.stats.skills) incoming.stats.skills = sb.stats.skills;
|
||||
if(sb.stats && sb.stats.snippet) incoming.stats.snippet = sb.stats.snippet;
|
||||
if(sb.snippet) incoming.stats.snippet = sb.snippet;
|
||||
// preserve api id
|
||||
if(sb.id) incoming.apiId = sb.id;
|
||||
|
||||
if(idx === -1){
|
||||
results.push(incoming);
|
||||
added++;
|
||||
} else {
|
||||
const prev = results[idx] || {};
|
||||
// merge with preference: prefer incoming.stats.* for movement/wounds/profile if present
|
||||
const merged = mergeObjects(prev, incoming);
|
||||
// ensure movement/wounds come from incoming when present
|
||||
if(incoming.stats){
|
||||
merged.stats = merged.stats || {};
|
||||
if(incoming.stats.movement) merged.stats.movement = incoming.stats.movement;
|
||||
if(incoming.stats.wounds) merged.stats.wounds = incoming.stats.wounds;
|
||||
if(incoming.stats.profile) merged.stats.profile = incoming.stats.profile;
|
||||
}
|
||||
results[idx] = merged;
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
|
||||
bestiaryObj.generatedAt = new Date().toISOString();
|
||||
bestiaryObj.count = results.length;
|
||||
|
||||
fs.writeFileSync(BESTIARY_PATH, JSON.stringify(bestiaryObj, null, 2));
|
||||
console.log(`Done. Added: ${added}, Updated: ${updated}. Total entries: ${results.length}`);
|
||||
}
|
||||
|
||||
if(require.main === module) main().catch(err => { console.error(err); process.exit(1); });
|
||||
110
scripts/import-bestiary-deathwatch.js
Normal file
110
scripts/import-bestiary-deathwatch.js
Normal file
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const https = require('https');
|
||||
|
||||
const BESTIARY_URL = 'https://www.40krpgtools.com/bestiary/';
|
||||
const DB_PATH = path.resolve(__dirname, '..', 'database', 'deathwatch-enemies-highconfidence.json');
|
||||
const OUT_PATH = path.resolve(__dirname, '..', 'database', 'deathwatch-enemies-bestiary-derived.json');
|
||||
const DATA_DIR = path.resolve(__dirname, '..', 'data');
|
||||
|
||||
function normalizeName(s) {
|
||||
return (s||'').toString().toLowerCase()
|
||||
.replace(/[\u2018\u2019\u201c\u201d]/g,'')
|
||||
.replace(/[^a-z0-9 ]+/g,' ')
|
||||
.replace(/\s+/g,' ').trim();
|
||||
}
|
||||
function levenshtein(a, b) {
|
||||
a = a || '';
|
||||
b = b || '';
|
||||
const m = a.length, n = b.length;
|
||||
if (!m) return n;
|
||||
if (!n) return m;
|
||||
const dp = Array.from({length: m+1}, () => Array(n+1).fill(0));
|
||||
for (let i=0;i<=m;i++) dp[i][0]=i;
|
||||
for (let j=0;j<=n;j++) dp[0][j]=j;
|
||||
for (let i=1;i<=m;i++){
|
||||
for (let j=1;j<=n;j++){
|
||||
const cost = a[i-1]===b[j-1] ? 0 : 1;
|
||||
dp[i][j] = Math.min(dp[i-1][j]+1, dp[i][j-1]+1, dp[i-1][j-1]+cost);
|
||||
}
|
||||
}
|
||||
return dp[m][n];
|
||||
}
|
||||
|
||||
function fetch(url, timeout=20000){
|
||||
return new Promise((resolve,reject)=>{
|
||||
const req = https.get(url, { timeout }, res => {
|
||||
if (res.statusCode && res.statusCode>=400) return reject(new Error('HTTP '+res.statusCode));
|
||||
let s=''; res.setEncoding('utf8'); res.on('data',d=>s+=d); res.on('end',()=>resolve(s));
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.on('timeout', ()=>req.destroy(new Error('timeout')));
|
||||
});
|
||||
}
|
||||
|
||||
function stripTags(s){ return s.replace(/<[^>]+>/g,' ').replace(/\s+/g,' ').trim(); }
|
||||
function parseBestiary(html){
|
||||
const rows=[];
|
||||
const trRe = /<tr[^>]*>([\s\S]*?)<\/tr>/gi; let m; while((m=trRe.exec(html))){
|
||||
const tr=m[1]; const tdRe=/<td[^>]*>([\s\S]*?)<\/td>/gi; let mm; const cols=[]; while((mm=tdRe.exec(tr))){ cols.push(stripTags(mm[1])); }
|
||||
if(cols.length>=1){ const name=cols[0]||''; const page=cols[cols.length-1]||''; const book=cols[cols.length-2]||''; if(name) rows.push({name,book,page}); }
|
||||
}
|
||||
if(rows.length===0){ // fallback
|
||||
const lines = html.split('\n');
|
||||
for(const line of lines){ if(/\|/.test(line)){ const parts=line.replace(/<[^>]+>/g,'').split('|').map(p=>p.trim()).filter(Boolean); if(parts.length>=3){ rows.push({name:parts[0], book:parts[parts.length-2], page:parts[parts.length-1]}); } } }
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function bestMatchFor(name, db){
|
||||
const n = normalizeName(name);
|
||||
for(const e of db){ if(normalizeName(e.name)===n) return {score:0, entry:e, method:'exact'}; }
|
||||
for(const e of db){ const cand=normalizeName(e.name); if(cand.includes(n) || n.includes(cand)) return {score:1, entry:e, method:'substr'}; }
|
||||
let best=null; for(const e of db){ const cand=normalizeName(e.name); const d=levenshtein(n,cand); if(!best||d<best.d) best={d,entry:e}; }
|
||||
if(best) return {score:best.d, entry:best.entry, method:'lev'}; return null;
|
||||
}
|
||||
|
||||
async function main(){
|
||||
if(!fs.existsSync(DB_PATH)){ console.error('highconfidence DB not found at',DB_PATH); process.exit(2); }
|
||||
const db = JSON.parse(fs.readFileSync(DB_PATH,'utf8'));
|
||||
console.log('Loaded highconfidence DB entries:', db.length);
|
||||
const localPdfFiles = fs.existsSync(DATA_DIR) ? fs.readdirSync(DATA_DIR).filter(f=>/\.pdf$/i.test(f)) : [];
|
||||
const localNames = localPdfFiles.map(f=>f.replace(/\.pdf$/i,''));
|
||||
console.log('Local PDFs found:', localPdfFiles.length);
|
||||
|
||||
console.log('Fetching bestiary...');
|
||||
const html = await fetch(BESTIARY_URL);
|
||||
const rows = parseBestiary(html);
|
||||
console.log('Total bestiary rows parsed:', rows.length);
|
||||
|
||||
// filter bestiary to those referencing Deathwatch books or matching local filenames
|
||||
const filtered = rows.filter(r=>{
|
||||
const book = (r.book||'').toLowerCase();
|
||||
if(book.includes('deathwatch')) return true;
|
||||
// if book matches any local pdf base name
|
||||
for(const ln of localNames){ if(book.includes(ln.toLowerCase()) || (r.name||'').toLowerCase().includes(ln.toLowerCase())) return true; }
|
||||
return false;
|
||||
});
|
||||
console.log('Filtered bestiary rows (deathwatch/local):', filtered.length);
|
||||
|
||||
// If the user expected 372, and filtered length differs, we'll still proceed but print note
|
||||
if(filtered.length!==372) console.log('Note: filtered count != 372 (found',filtered.length,')');
|
||||
|
||||
const matches=[]; const unmatched=[];
|
||||
for(const be of filtered){ const m = bestMatchFor(be.name, db); if(m && m.entry){ // prepare structured output
|
||||
const e = m.entry; // pick fields
|
||||
const localSources = (e.sources||[]).filter(s=> localPdfFiles.some(lp=> s.toLowerCase().includes(lp.toLowerCase()) || lp.toLowerCase().includes(s.toLowerCase())) );
|
||||
matches.push({bestiaryName: be.name, bestiaryBook: be.book, bestiaryPage: be.page, matchedName: e.name, score:m.score, method:m.method, wounds: e.wounds||null, toughness: e.toughness||null, armour: e.armour||null, armourByLoc: e.armourByLoc||null, sources: e.sources||[], pages: e.pages||[], localSources});
|
||||
} else {
|
||||
unmatched.push(be);
|
||||
}
|
||||
}
|
||||
|
||||
const summary = {filtered: filtered.length, matched: matches.length, unmatched: unmatched.length};
|
||||
const out = {summary, matches, unmatched};
|
||||
fs.writeFileSync(OUT_PATH, JSON.stringify(out,null,2));
|
||||
console.log('Wrote', OUT_PATH, 'summary:', summary);
|
||||
}
|
||||
|
||||
if(require.main===module) main().catch(err=>{ console.error(err); process.exit(1); });
|
||||
0
scripts/import-remote-unmatched.js
Normal file
0
scripts/import-remote-unmatched.js
Normal file
147
scripts/match-bestiary-to-pdfs.js
Normal file
147
scripts/match-bestiary-to-pdfs.js
Normal file
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const https = require('https');
|
||||
// lightweight HTML parsing (avoid loading cheerio to prevent extraneous deps)
|
||||
|
||||
|
||||
const BESTIARY_URL = 'https://www.40krpgtools.com/bestiary/';
|
||||
const DB_PATH = path.resolve(__dirname, '..', 'database', 'deathwatch-enemies-highconfidence.json');
|
||||
const OUT_PATH = path.resolve(__dirname, '..', 'database', 'bestiary-matches.json');
|
||||
|
||||
function normalizeName(s) {
|
||||
return (s||'').toString().toLowerCase()
|
||||
.replace(/[\u2018\u2019\u201c\u201d]/g,'')
|
||||
.replace(/[^a-z0-9 ]+/g,' ')
|
||||
.replace(/\s+/g,' ').trim();
|
||||
}
|
||||
|
||||
function levenshtein(a, b) {
|
||||
a = a || '';
|
||||
b = b || '';
|
||||
const m = a.length, n = b.length;
|
||||
if (!m) return n;
|
||||
if (!n) return m;
|
||||
const dp = Array.from({length: m+1}, () => Array(n+1).fill(0));
|
||||
for (let i=0;i<=m;i++) dp[i][0]=i;
|
||||
for (let j=0;j<=n;j++) dp[0][j]=j;
|
||||
for (let i=1;i<=m;i++){
|
||||
for (let j=1;j<=n;j++){
|
||||
const cost = a[i-1]===b[j-1] ? 0 : 1;
|
||||
dp[i][j] = Math.min(dp[i-1][j]+1, dp[i][j-1]+1, dp[i-1][j-1]+cost);
|
||||
}
|
||||
}
|
||||
return dp[m][n];
|
||||
}
|
||||
|
||||
async function fetchBestiary() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.get(BESTIARY_URL, { timeout: 20000 }, (res) => {
|
||||
if (res.statusCode && res.statusCode >= 400) return reject(new Error('HTTP '+res.statusCode));
|
||||
let buf = '';
|
||||
res.setEncoding('utf8');
|
||||
res.on('data', d => buf += d);
|
||||
res.on('end', () => resolve(buf));
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.on('timeout', () => { req.destroy(new Error('timeout')); });
|
||||
});
|
||||
}
|
||||
|
||||
function stripTags(s) {
|
||||
return s.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function parseBestiary(html) {
|
||||
const rows = [];
|
||||
// extract all table row contents
|
||||
const trRe = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
|
||||
let m;
|
||||
while ((m = trRe.exec(html)) !== null) {
|
||||
const tr = m[1];
|
||||
const tdRe = /<td[^>]*>([\s\S]*?)<\/td>/gi;
|
||||
const cols = [];
|
||||
let mm;
|
||||
while ((mm = tdRe.exec(tr)) !== null) {
|
||||
cols.push(stripTags(mm[1]));
|
||||
}
|
||||
if (cols.length >= 1) {
|
||||
const name = cols[0] || '';
|
||||
const page = cols[cols.length-1] || '';
|
||||
const book = cols[cols.length-2] || '';
|
||||
if (name) rows.push({name, book, page});
|
||||
}
|
||||
}
|
||||
// fallback: look for pipe table lines
|
||||
if (rows.length === 0) {
|
||||
const lines = html.split('\n');
|
||||
for (const line of lines) {
|
||||
if (/\|/.test(line)) {
|
||||
const parts = line.replace(/<[^>]+>/g,'').split('|').map(p=>p.trim()).filter(Boolean);
|
||||
if (parts.length >= 3) {
|
||||
const name = parts[0];
|
||||
const page = parts[parts.length-1];
|
||||
const book = parts[parts.length-2];
|
||||
rows.push({name, book, page});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function bestMatchFor(name, db) {
|
||||
const n = normalizeName(name);
|
||||
// exact normalized match
|
||||
for (const e of db) {
|
||||
if (normalizeName(e.name) === n) return {score:0, entry:e, method:'exact'};
|
||||
}
|
||||
// substring match
|
||||
for (const e of db) {
|
||||
if (normalizeName(e.name).includes(n) || n.includes(normalizeName(e.name))) return {score:1, entry:e, method:'substr'};
|
||||
}
|
||||
// levenshtein best candidate
|
||||
let best = null;
|
||||
for (const e of db) {
|
||||
const cand = normalizeName(e.name);
|
||||
const d = levenshtein(n, cand);
|
||||
if (!best || d < best.d) best = {d, entry:e};
|
||||
}
|
||||
if (best) {
|
||||
return {score: best.d, entry: best.entry, method:'lev'};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function main(){
|
||||
console.log('Reading DB:', DB_PATH);
|
||||
if (!fs.existsSync(DB_PATH)) {
|
||||
console.error('DB file not found:', DB_PATH);
|
||||
process.exit(2);
|
||||
}
|
||||
const db = JSON.parse(fs.readFileSync(DB_PATH, 'utf8'));
|
||||
console.log('Fetching bestiary page...');
|
||||
const html = await fetchBestiary();
|
||||
const entries = parseBestiary(html);
|
||||
console.log('Parsed bestiary entries:', entries.length);
|
||||
const out = [];
|
||||
let matched = 0;
|
||||
for (const be of entries) {
|
||||
const m = bestMatchFor(be.name, db);
|
||||
const rec = {bestiaryName: be.name, book: be.book, page: be.page, matched: !!m, method: m?m.method:null};
|
||||
if (m && m.entry) {
|
||||
rec.matchName = m.entry.name;
|
||||
rec.matchSources = m.entry.sources || m.entry.sources || [];
|
||||
rec.matchPages = m.entry.pages || m.entry.pages || [];
|
||||
matched++;
|
||||
}
|
||||
out.push(rec);
|
||||
}
|
||||
const summary = {total: entries.length, matched, unmatched: entries.length-matched};
|
||||
fs.writeFileSync(OUT_PATH, JSON.stringify({summary, matches: out}, null, 2));
|
||||
console.log('Wrote matches to', OUT_PATH, 'summary:', summary);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch(err=>{ console.error(err); process.exit(1); });
|
||||
}
|
||||
91
scripts/merge-npcs-from-api.js
Normal file
91
scripts/merge-npcs-from-api.js
Normal file
@@ -0,0 +1,91 @@
|
||||
// merge-npcs-from-api.js
|
||||
// Fetches NPCs from local API and upserts them into the bestiary DB
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const http = require('http');
|
||||
|
||||
const API_URL = 'http://192.168.1.144:5000/api/npcs';
|
||||
const BESTIARY_PATH = path.join(__dirname, '../database/deathwatch-bestiary-extracted.json');
|
||||
const BACKUP_PATH = path.join(__dirname, `../database/deathwatch-bestiary-extracted.backup.${Date.now()}.json`);
|
||||
|
||||
function fetchAPI(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
http.get(url, (res) => {
|
||||
let data = '';
|
||||
res.on('data', chunk => data += chunk);
|
||||
res.on('end', () => {
|
||||
try {
|
||||
resolve(JSON.parse(data));
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeNPC(apiNpc) {
|
||||
// Map API NPC to bestiary format
|
||||
return {
|
||||
name: apiNpc.name,
|
||||
type: apiNpc.type,
|
||||
chapter: apiNpc.chapter,
|
||||
faction: apiNpc.faction,
|
||||
threat: apiNpc.threat_level,
|
||||
description: apiNpc.description,
|
||||
source: apiNpc.source,
|
||||
stats: {
|
||||
...apiNpc.stats
|
||||
},
|
||||
equipment: apiNpc.equipment,
|
||||
abilities: apiNpc.abilities,
|
||||
apiId: apiNpc.id
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// Backup bestiary
|
||||
fs.copyFileSync(BESTIARY_PATH, BACKUP_PATH);
|
||||
console.log('Backed up bestiary to', BACKUP_PATH);
|
||||
|
||||
// Read bestiary object
|
||||
let bestiaryObj = {};
|
||||
try {
|
||||
bestiaryObj = JSON.parse(fs.readFileSync(BESTIARY_PATH, 'utf8'));
|
||||
} catch (e) {
|
||||
console.error('Could not read bestiary, starting with empty results array.');
|
||||
bestiaryObj = { results: [] };
|
||||
}
|
||||
if (!Array.isArray(bestiaryObj.results)) bestiaryObj.results = [];
|
||||
const bestiary = bestiaryObj.results;
|
||||
|
||||
// Fetch API NPCs
|
||||
let apiNpcs;
|
||||
try {
|
||||
const apiData = await fetchAPI(API_URL);
|
||||
apiNpcs = apiData.npcs || [];
|
||||
} catch (e) {
|
||||
console.error('Failed to fetch from API:', e);
|
||||
return;
|
||||
}
|
||||
|
||||
let added = 0, updated = 0;
|
||||
for (const apiNpc of apiNpcs) {
|
||||
const norm = normalizeNPC(apiNpc);
|
||||
// Try to match by name or bestiaryName
|
||||
const idx = bestiary.findIndex(e => (e.name && e.name === norm.name) || (e.bestiaryName && e.bestiaryName === norm.name));
|
||||
if (idx === -1) {
|
||||
bestiary.push(norm);
|
||||
added++;
|
||||
} else {
|
||||
bestiary[idx] = { ...bestiary[idx], ...norm };
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
|
||||
bestiaryObj.count = bestiary.length;
|
||||
fs.writeFileSync(BESTIARY_PATH, JSON.stringify(bestiaryObj, null, 2));
|
||||
console.log(`NPCs merged. Added: ${added}, Updated: ${updated}`);
|
||||
}
|
||||
|
||||
main();
|
||||
211
scripts/merge-validate-upsert-npcs.js
Normal file
211
scripts/merge-validate-upsert-npcs.js
Normal file
@@ -0,0 +1,211 @@
|
||||
// merge-validate-upsert-npcs.js
|
||||
// Fetch NPCs from local API, normalize to bestiary schema, validate and upsert into database/deathwatch-bestiary-extracted.json
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const http = require('http');
|
||||
|
||||
const API_URL = 'http://192.168.1.144:5000/api/npcs';
|
||||
const BESTIARY_PATH = path.join(__dirname, '../database/deathwatch-bestiary-extracted.json');
|
||||
const BACKUP_PATH = path.join(__dirname, `../database/deathwatch-bestiary-extracted.backup.${Date.now()}.json`);
|
||||
|
||||
function fetchAPI(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
http.get(url, (res) => {
|
||||
let data = '';
|
||||
res.on('data', chunk => data += chunk);
|
||||
res.on('end', () => {
|
||||
try {
|
||||
resolve(JSON.parse(data));
|
||||
} catch (e) {
|
||||
reject(new Error('Failed to parse API response: ' + e.message + '\n' + data.slice(0,2000)));
|
||||
}
|
||||
});
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function asIntOrNull(v) {
|
||||
if (v === null || v === undefined) return null;
|
||||
const n = parseInt(v, 10);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
function normalizeApiNpc(apiNpc) {
|
||||
const s = apiNpc.stats || {};
|
||||
const profile = {
|
||||
ws: asIntOrNull(s.ws),
|
||||
bs: asIntOrNull(s.bs),
|
||||
s: asIntOrNull(s.s),
|
||||
t: asIntOrNull(s.t),
|
||||
ag: asIntOrNull(s.ag),
|
||||
int: asIntOrNull(s.int),
|
||||
per: asIntOrNull(s.per),
|
||||
wp: asIntOrNull(s.wp),
|
||||
fel: asIntOrNull(s.fel),
|
||||
modifiers: null
|
||||
};
|
||||
|
||||
// compact movement if present
|
||||
const movement = s.movement || null;
|
||||
const wounds = asIntOrNull(s.wounds);
|
||||
const armour = (s.armor || s.armour) ? String(s.armor || s.armour) : null;
|
||||
|
||||
// equipment array -> gear string
|
||||
const gearArr = Array.isArray(apiNpc.equipment) ? apiNpc.equipment : [];
|
||||
const gear = gearArr.length ? gearArr.join('; ') : null;
|
||||
|
||||
// try to pick out obvious weapons from equipment by keywords
|
||||
const weaponsCandidates = gearArr.filter(it => /bolt|las|plasma|sword|knife|axe|grenade|blade|pistol|rifle|power|monoblade|stikk|autogun/i.test(it));
|
||||
const weapons = weaponsCandidates.length ? weaponsCandidates.join('; ') : null;
|
||||
|
||||
const normalized = {
|
||||
bestiaryName: apiNpc.name || apiNpc.bestiaryName || null,
|
||||
book: apiNpc.source || null,
|
||||
page: apiNpc.page || null,
|
||||
pdf: apiNpc.source ? apiNpc.source.replace(/[:\s]/g, '_') + '.pdf' : null,
|
||||
pageText: null,
|
||||
// preserve some meta
|
||||
source: apiNpc.source || null,
|
||||
apiId: apiNpc.id || null,
|
||||
description: apiNpc.description || null,
|
||||
type: apiNpc.type || null,
|
||||
faction: apiNpc.faction || null,
|
||||
abilities: Array.isArray(apiNpc.abilities) ? apiNpc.abilities : null,
|
||||
stats: {
|
||||
profile,
|
||||
movement,
|
||||
wounds,
|
||||
toughness: null,
|
||||
skills: null,
|
||||
talents: null,
|
||||
traits: null,
|
||||
armour,
|
||||
weapons,
|
||||
gear,
|
||||
snippet: null,
|
||||
chosenOffset: null
|
||||
}
|
||||
};
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function mergeRecords(existing, incoming) {
|
||||
// shallow merge: prefer existing non-null values, otherwise take incoming
|
||||
const out = { ...existing };
|
||||
|
||||
// top-level fields
|
||||
['bestiaryName','book','page','pdf','pageText','source','apiId','description','type','faction','abilities'].forEach(k=>{
|
||||
if ((!out[k] || out[k] === null) && incoming[k] !== undefined) out[k] = incoming[k];
|
||||
});
|
||||
|
||||
// merge stats
|
||||
out.stats = out.stats || {};
|
||||
const ex = existing.stats || {};
|
||||
const inS = incoming.stats || {};
|
||||
|
||||
// profile
|
||||
out.stats.profile = out.stats.profile || {};
|
||||
const exP = ex.profile || {};
|
||||
const inP = inS.profile || {};
|
||||
['ws','bs','s','t','ag','int','per','wp','fel','modifiers'].forEach(k=>{
|
||||
const exV = exP[k];
|
||||
const inV = inP[k];
|
||||
out.stats.profile[k] = (exV !== null && exV !== undefined) ? exV : (inV !== null && inV !== undefined ? inV : null);
|
||||
});
|
||||
|
||||
// other stat fields
|
||||
['movement','wounds','toughness','skills','talents','traits','armour','weapons','gear','snippet','chosenOffset'].forEach(k=>{
|
||||
const exV = (ex && ex[k] !== undefined) ? ex[k] : undefined;
|
||||
const inV = (inS && inS[k] !== undefined) ? inS[k] : undefined;
|
||||
out.stats[k] = (exV !== undefined && exV !== null) ? exV : (inV !== undefined ? inV : null);
|
||||
});
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// backup bestiary
|
||||
if (fs.existsSync(BESTIARY_PATH)) {
|
||||
fs.copyFileSync(BESTIARY_PATH, BACKUP_PATH);
|
||||
console.log('Backed up bestiary to', BACKUP_PATH);
|
||||
} else {
|
||||
console.log('No existing bestiary found at', BESTIARY_PATH, ' - a new file will be created');
|
||||
}
|
||||
|
||||
let bestiaryObj = { results: [] };
|
||||
try {
|
||||
if (fs.existsSync(BESTIARY_PATH)) {
|
||||
bestiaryObj = JSON.parse(fs.readFileSync(BESTIARY_PATH, 'utf8'));
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to read existing bestiary:', e.message);
|
||||
bestiaryObj = { results: [] };
|
||||
}
|
||||
if (!Array.isArray(bestiaryObj.results)) bestiaryObj.results = [];
|
||||
|
||||
// fetch API NPCs
|
||||
let apiNpcs = [];
|
||||
try {
|
||||
const j = await fetchAPI(API_URL);
|
||||
apiNpcs = j.npcs || [];
|
||||
} catch (e) {
|
||||
console.error('Failed to fetch API NPCs:', e.message);
|
||||
return;
|
||||
}
|
||||
|
||||
const added = [];
|
||||
const updated = [];
|
||||
|
||||
for (const apiNpc of apiNpcs) {
|
||||
const norm = normalizeApiNpc(apiNpc);
|
||||
|
||||
// find existing by bestiaryName or apiId or name
|
||||
const idx = bestiaryObj.results.findIndex(r => {
|
||||
if (!r) return false;
|
||||
if (r.apiId && norm.apiId && r.apiId === norm.apiId) return true;
|
||||
const rn = (r.bestiaryName || r.name || '').toString().trim().toLowerCase();
|
||||
const nn = (norm.bestiaryName || '').toString().trim().toLowerCase();
|
||||
return rn && nn && rn === nn;
|
||||
});
|
||||
|
||||
if (idx === -1) {
|
||||
// new entry
|
||||
// ensure top-level required keys exist
|
||||
const toInsert = {
|
||||
bestiaryName: norm.bestiaryName,
|
||||
book: norm.book,
|
||||
page: norm.page,
|
||||
pdf: norm.pdf,
|
||||
pageText: norm.pageText,
|
||||
stats: norm.stats,
|
||||
description: norm.description,
|
||||
source: norm.source,
|
||||
apiId: norm.apiId,
|
||||
type: norm.type,
|
||||
faction: norm.faction,
|
||||
abilities: norm.abilities || null,
|
||||
insertedAt: new Date().toISOString()
|
||||
};
|
||||
bestiaryObj.results.push(toInsert);
|
||||
added.push(norm.bestiaryName || norm.apiId || 'unknown');
|
||||
} else {
|
||||
const existing = bestiaryObj.results[idx];
|
||||
const merged = mergeRecords(existing, norm);
|
||||
merged.updatedAt = new Date().toISOString();
|
||||
bestiaryObj.results[idx] = merged;
|
||||
updated.push(merged.bestiaryName || merged.apiId || 'unknown');
|
||||
}
|
||||
}
|
||||
|
||||
bestiaryObj.generatedAt = new Date().toISOString();
|
||||
bestiaryObj.count = bestiaryObj.results.length;
|
||||
|
||||
fs.writeFileSync(BESTIARY_PATH, JSON.stringify(bestiaryObj, null, 2));
|
||||
console.log(`Merge complete. Added: ${added.length}, Updated: ${updated.length}`);
|
||||
if (added.length) console.log('Added:', added.join(', '));
|
||||
if (updated.length) console.log('Updated:', updated.join(', '));
|
||||
}
|
||||
|
||||
main();
|
||||
136
scripts/normalize-and-merge-bestiary.js
Normal file
136
scripts/normalize-and-merge-bestiary.js
Normal file
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const DB_PATH = path.resolve(__dirname, '../database/deathwatch-bestiary-extracted.json');
|
||||
const BACKUPS_DIR = path.resolve(__dirname, '../database/backups');
|
||||
const PDFJS_PATH = path.resolve(__dirname, '../database/alexei-pdfjs.json');
|
||||
|
||||
function readJson(p) {
|
||||
return JSON.parse(fs.readFileSync(p, 'utf8'));
|
||||
}
|
||||
|
||||
function writeJson(p, obj) {
|
||||
fs.writeFileSync(p, JSON.stringify(obj, null, 2), 'utf8');
|
||||
}
|
||||
|
||||
function ensureDir(p) {
|
||||
if (!fs.existsSync(p)) fs.mkdirSync(p, { recursive: true });
|
||||
}
|
||||
|
||||
function extractMovement(m) {
|
||||
if (!m || typeof m !== 'string') return null;
|
||||
const movementMatch = m.match(/(\d+\/\d+\/\d+\/\d+)/);
|
||||
if (movementMatch) return movementMatch[1];
|
||||
// fallback: look for patterns like 3/6/9/18 without capture groups
|
||||
const alt = m.match(/(\d+\s*\/\s*\d+\s*\/\s*\d+\s*\/\s*\d+)/);
|
||||
if (alt) return alt[1].replace(/\s+/g, '');
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractWounds(s) {
|
||||
if (s == null) return null;
|
||||
if (typeof s === 'number') return s;
|
||||
const str = String(s);
|
||||
const m = str.match(/Wounds:\s*(\d+)/i) || str.match(/(\d{1,3})\s*$/);
|
||||
if (m) return parseInt(m[1], 10);
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeEntry(entry, pdfjsResults) {
|
||||
let changed = false;
|
||||
if (!entry.stats) entry.stats = {};
|
||||
|
||||
// Normalize movement
|
||||
const movementRaw = entry.stats.movement;
|
||||
const mov = extractMovement(movementRaw);
|
||||
if (mov && mov !== movementRaw) {
|
||||
entry.stats.movement = mov;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
// Normalize wounds
|
||||
const woundsRaw = entry.stats.wounds;
|
||||
const wounds = extractWounds(woundsRaw || movementRaw);
|
||||
if (wounds != null && entry.stats.wounds !== wounds) {
|
||||
entry.stats.wounds = wounds;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
// If profile missing or incomplete, try to fill from pdfjs artifacts
|
||||
const needProfile = !entry.stats.profile || Object.keys(entry.stats.profile).length < 5;
|
||||
const needWeapons = !entry.stats.weapons && !entry.stats.gear;
|
||||
|
||||
if ((needProfile || needWeapons) && pdfjsResults && entry.pdf) {
|
||||
const pageNum = parseInt(entry.page, 10);
|
||||
const match = pdfjsResults.find(r => {
|
||||
if (!r || !r.pdf) return false;
|
||||
if (String(r.pdf).trim() !== String(entry.pdf).trim()) return false;
|
||||
if (r.foundPage && pageNum && r.foundPage === pageNum) return true;
|
||||
if (Array.isArray(r.range) && pageNum && pageNum >= r.range[0] && pageNum <= r.range[1]) return true;
|
||||
if (r.chosenPage && pageNum && r.chosenPage === pageNum) return true;
|
||||
return false;
|
||||
});
|
||||
|
||||
if (match && match.stats) {
|
||||
// copy profile if missing
|
||||
if (needProfile && match.stats.profile) {
|
||||
entry.stats.profile = entry.stats.profile || {};
|
||||
Object.assign(entry.stats.profile, match.stats.profile);
|
||||
changed = true;
|
||||
}
|
||||
// copy movement/wounds/weapons/gear if missing
|
||||
if (!entry.stats.movement && match.stats.movement) {
|
||||
entry.stats.movement = match.stats.movement;
|
||||
changed = true;
|
||||
}
|
||||
if ((!entry.stats.wounds || entry.stats.wounds === null) && match.stats.wounds) {
|
||||
entry.stats.wounds = match.stats.wounds;
|
||||
changed = true;
|
||||
}
|
||||
if (needWeapons && (match.stats.weapons || match.stats.gear)) {
|
||||
entry.stats.weapons = entry.stats.weapons || match.stats.weapons || null;
|
||||
entry.stats.gear = entry.stats.gear || match.stats.gear || null;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return changed;
|
||||
}
|
||||
|
||||
function main() {
|
||||
console.log('Loading bestiary:', DB_PATH);
|
||||
const db = readJson(DB_PATH);
|
||||
let pdfjs = null;
|
||||
try {
|
||||
pdfjs = readJson(PDFJS_PATH);
|
||||
} catch (e) {
|
||||
console.warn('Could not read pdfjs artifacts:', PDFJS_PATH);
|
||||
}
|
||||
|
||||
ensureDir(BACKUPS_DIR);
|
||||
const backupPath = path.join(BACKUPS_DIR, `deathwatch-bestiary-extracted.json.backup.${Date.now()}.json`);
|
||||
fs.copyFileSync(DB_PATH, backupPath);
|
||||
console.log('Backed up DB to', backupPath);
|
||||
|
||||
const results = db.results || [];
|
||||
let modified = 0;
|
||||
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const entry = results[i];
|
||||
const changed = normalizeEntry(entry, pdfjs && pdfjs.results ? pdfjs.results : (pdfjs && pdfjs.matches ? pdfjs.matches : null));
|
||||
if (changed) modified++;
|
||||
}
|
||||
|
||||
if (modified > 0) {
|
||||
db.generatedAt = new Date().toISOString();
|
||||
db.count = results.length;
|
||||
writeJson(DB_PATH, db);
|
||||
console.log(`Updated ${modified} entries and wrote DB (count=${db.count})`);
|
||||
} else {
|
||||
console.log('No changes necessary');
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
67
scripts/normalize-bestiary.js
Normal file
67
scripts/normalize-bestiary.js
Normal file
@@ -0,0 +1,67 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
function backup(filePath) {
|
||||
const ts = Date.now();
|
||||
const bakPath = filePath + `.backup.${ts}.json`;
|
||||
fs.copyFileSync(filePath, bakPath);
|
||||
return bakPath;
|
||||
}
|
||||
|
||||
function normalizeFile(filePath) {
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
const data = JSON.parse(raw);
|
||||
if (!Array.isArray(data.results)) {
|
||||
console.error('No results array in', filePath);
|
||||
return { updated: 0, total: 0 };
|
||||
}
|
||||
const total = data.results.length;
|
||||
let updated = 0;
|
||||
|
||||
const mapped = data.results.map((e) => {
|
||||
const out = Object.assign({}, e);
|
||||
if (e.stats && typeof e.stats === 'object') {
|
||||
const s = e.stats;
|
||||
const fields = ['profile','wounds','movement','toughness','skills','talents','traits','armour','weapons','gear','snippet'];
|
||||
let any = false;
|
||||
fields.forEach(f => {
|
||||
if (s[f] !== undefined && out[f] === undefined) {
|
||||
out[f] = s[f];
|
||||
any = true;
|
||||
}
|
||||
});
|
||||
if (any) updated++;
|
||||
}
|
||||
return out;
|
||||
});
|
||||
|
||||
// update count if present
|
||||
if (typeof data.count === 'number') data.count = mapped.length;
|
||||
data.results = mapped;
|
||||
|
||||
const bak = backup(filePath);
|
||||
fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8');
|
||||
console.log(`Normalized ${filePath}: ${updated}/${total} entries updated. Backup created at ${bak}`);
|
||||
return { updated, total, backup: bak };
|
||||
}
|
||||
|
||||
function main() {
|
||||
const repoRoot = path.resolve(__dirname, '..');
|
||||
const targets = [
|
||||
path.join(repoRoot, 'database', 'deathwatch-bestiary-extracted.json'),
|
||||
path.join(repoRoot, 'public', 'deathwatch-bestiary-extracted.json'),
|
||||
];
|
||||
targets.forEach(tp => {
|
||||
if (fs.existsSync(tp)) {
|
||||
try {
|
||||
normalizeFile(tp);
|
||||
} catch (err) {
|
||||
console.error('Error normalizing', tp, err);
|
||||
}
|
||||
} else {
|
||||
console.warn('File not found, skipping:', tp);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
||||
44
scripts/parse-alexei-pdfjs-results.js
Normal file
44
scripts/parse-alexei-pdfjs-results.js
Normal file
@@ -0,0 +1,44 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const dataPath = path.resolve(__dirname, '../database/alexei-pdfjs.json');
|
||||
if (!fs.existsSync(dataPath)) {
|
||||
console.error('missing file:', dataPath);
|
||||
process.exit(2);
|
||||
}
|
||||
const j = JSON.parse(fs.readFileSync(dataPath,'utf8'));
|
||||
const findings = j.findings || [];
|
||||
let hit = null;
|
||||
for (const f of findings) {
|
||||
if (/(alexei|drahj)/i.test(f.text)) { hit = f; break; }
|
||||
}
|
||||
if (!hit) {
|
||||
console.error('No finding with alexei/drahj found');
|
||||
process.exit(1);
|
||||
}
|
||||
const txt = hit.text;
|
||||
// narrow to area around the name
|
||||
const idx = txt.toLowerCase().indexOf('alexei');
|
||||
const windowText = txt.substr(Math.max(0, idx-200), 800);
|
||||
function findMovement(s){
|
||||
const m = s.match(/Movement:\s*([0-9\/]+(?:\/[0-9]+)*)/i);
|
||||
return m ? m[1].trim() : null;
|
||||
}
|
||||
function findWounds(s){
|
||||
const m = s.match(/Wounds:\s*(\d{1,3})/i);
|
||||
return m ? parseInt(m[1],10) : null;
|
||||
}
|
||||
function findProfile(s){
|
||||
// look for 9 small integers in a row (allow parentheses)
|
||||
const m = s.match(/(\(?\d{1,2}\)?(?:\s+\(?\d{1,2}\)?){8})/);
|
||||
if (!m) return null;
|
||||
const nums = m[1].replace(/[()]/g,'').trim().split(/\s+/).map(n=>parseInt(n,10));
|
||||
if (nums.length!==9) return null;
|
||||
const keys = ['ws','bs','s','t','ag','int','per','wp','fel'];
|
||||
const obj = {};
|
||||
keys.forEach((k,i)=>obj[k]=nums[i]);
|
||||
return obj;
|
||||
}
|
||||
const movement = findMovement(windowText);
|
||||
const wounds = findWounds(windowText);
|
||||
const profile = findProfile(windowText);
|
||||
console.log(JSON.stringify({ page: hit.page, range: hit.page? [Math.max(1, hit.page-2), hit.page+2] : null, movement, wounds, profile, snippet: windowText.replace(/\n+/g,' ').slice(0,800) }, null, 2));
|
||||
107
scripts/parse-alexei.js
Normal file
107
scripts/parse-alexei.js
Normal file
@@ -0,0 +1,107 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const pdf = require('pdf-parse');
|
||||
|
||||
const DATA_DIR = path.resolve(__dirname, '..', 'data');
|
||||
const targetPdf = 'Deathwatch - The Emperor Protects.pdf';
|
||||
const name = /alexei\s+drahj/i;
|
||||
|
||||
function extractStatsFromText(text){
|
||||
const out={profile:null,movement:null,wounds:null,toughness:null,skills:null,talents:null,traits:null,armour:null,weapons:null,gear:null,snippet:null};
|
||||
if(!text) return out;
|
||||
const t = text.replace(/\r/g,'\n');
|
||||
const lines = t.split('\n').map(l=>l.trim()).filter(Boolean);
|
||||
const joined = lines.join('\n');
|
||||
|
||||
// Profile
|
||||
let prof = null;
|
||||
const profHeader = joined.match(/ws\W*bs\W*s\W*t\W*ag\W*int\W*per\W*wp\W*fel/i);
|
||||
if(profHeader){
|
||||
const after = joined.slice(profHeader.index + profHeader[0].length, profHeader.index + profHeader[0].length + 300);
|
||||
const nums = (after.match(/\d{1,3}/g) || []).slice(0,9);
|
||||
if(nums.length>=9) prof = nums.map(n=>parseInt(n,10));
|
||||
} else {
|
||||
const pMatch = joined.match(/profile[\s\S]{0,200}?(\d{1,3}[\s\S]*?\d{1,3})/i);
|
||||
if(pMatch){ const nums = (pMatch[0].match(/\d{1,3}/g)||[]).slice(0,9); if(nums.length>=9) prof = nums.map(n=>parseInt(n,10)); }
|
||||
}
|
||||
if(prof) out.profile = { ws:prof[0], bs:prof[1], s:prof[2], t:prof[3], ag:prof[4], int:prof[5], per:prof[6], wp:prof[7], fel:prof[8] };
|
||||
|
||||
// Movement
|
||||
const mv = joined.match(/movement[:\s]*([0-9]+\/[0-9]+\/[0-9]+\/[0-9]+)/i) || joined.match(/\b(\d+\/\d+\/\d+\/\d+)\b/);
|
||||
if(mv) out.movement = mv[1] ? mv[1].trim() : mv[0].trim();
|
||||
|
||||
// Wounds
|
||||
const wMatch = joined.match(/wounds?[:\s]*([0-9]{1,3})/i);
|
||||
if(wMatch) out.wounds = parseInt(wMatch[1],10);
|
||||
|
||||
// Toughness
|
||||
const toughMatch = joined.match(/toughness[:\s]*([0-9]{1,3})/i);
|
||||
if(toughMatch) out.toughness = parseInt(toughMatch[1],10);
|
||||
|
||||
function captureSectionByHeaders(text, label){
|
||||
const re = new RegExp(label+':?\s*([\s\S]*?)(?=\n(?:Skills|Talents|Traits|Armou?r|Weapons|Gear|$):?)','i');
|
||||
const m = text.match(re);
|
||||
if(!m) return null;
|
||||
return m[1].replace(/\n+/g,' ').replace(/\s+/g,' ').trim();
|
||||
}
|
||||
|
||||
out.skills = captureSectionByHeaders(joined, 'Skills');
|
||||
out.talents = captureSectionByHeaders(joined, 'Talents');
|
||||
out.traits = captureSectionByHeaders(joined, 'Traits');
|
||||
out.armour = captureSectionByHeaders(joined, 'Armour') || captureSectionByHeaders(joined, 'Armor');
|
||||
out.weapons = captureSectionByHeaders(joined, 'Weapons');
|
||||
out.gear = captureSectionByHeaders(joined, 'Gear');
|
||||
|
||||
out.snippet = lines.slice(0,8).join(' ').replace(/\s+/g,' ').trim().slice(0,800);
|
||||
return out;
|
||||
}
|
||||
|
||||
(async function(){
|
||||
const pdfPath = path.join(DATA_DIR, targetPdf);
|
||||
if(!fs.existsSync(pdfPath)){ console.error('PDF not found:', pdfPath); process.exit(1); }
|
||||
const data = fs.readFileSync(pdfPath);
|
||||
const parsed = await pdf(data);
|
||||
const pages = (parsed.text||'').split('\f');
|
||||
const matches = [];
|
||||
for(let i=0;i<pages.length;i++){
|
||||
if(name.test(pages[i])){
|
||||
// search surrounding pages for best stat block
|
||||
const candidates = [];
|
||||
for(let off = -5; off<=5; off++){
|
||||
const idx = i + off;
|
||||
if(idx < 0 || idx >= pages.length) continue;
|
||||
const txt = pages[idx];
|
||||
const stats = extractStatsFromText(txt);
|
||||
// heuristic score
|
||||
let score = 0;
|
||||
if(stats.wounds) score += 5;
|
||||
if(stats.movement) score += 4;
|
||||
if(stats.profile) score += 6;
|
||||
if(stats.skills) score += 2;
|
||||
if(stats.talents) score += 2;
|
||||
if(stats.traits) score += 2;
|
||||
// small bonus for presence of 'Transpired' or name tokens
|
||||
if(/transpired/i.test(txt)) score += 1;
|
||||
candidates.push({idx, off, stats, score, snippet: stats.snippet});
|
||||
}
|
||||
candidates.sort((a,b)=>b.score - a.score);
|
||||
const best = candidates[0];
|
||||
if(best){
|
||||
// combine surrounding pages for more robust parsing (handle broken headers)
|
||||
const start = Math.max(0, best.idx-2);
|
||||
const end = Math.min(pages.length-1, best.idx+2);
|
||||
const combined = pages.slice(start, end+1).join('\n\f\n');
|
||||
const refined = extractStatsFromText(combined);
|
||||
// carry over chosen metadata
|
||||
refined.chosenPageIndex = best.idx;
|
||||
refined.chosenPageNumberApprox = best.idx+1;
|
||||
refined.chosenOffset = best.off;
|
||||
matches.push({pdf:targetPdf,pageIndex:best.idx,pageNumber:best.idx+1,snippet:refined.snippet,stats:refined});
|
||||
}
|
||||
}
|
||||
}
|
||||
const outPath = path.resolve(__dirname, '..', 'database', 'alexei-parse.json');
|
||||
fs.writeFileSync(outPath, JSON.stringify({generatedAt:new Date().toISOString(), matches},null,2));
|
||||
console.log('Wrote', outPath, 'matches:', matches.length);
|
||||
if(matches.length) console.log(JSON.stringify(matches[0],null,2));
|
||||
})();
|
||||
76
scripts/prune-bestiary.js
Normal file
76
scripts/prune-bestiary.js
Normal file
@@ -0,0 +1,76 @@
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const DB_PATH = path.join(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json')
|
||||
const BACKUP_DIR = path.join(__dirname, '..', 'database', 'backups')
|
||||
|
||||
function loadDb() {
|
||||
return JSON.parse(fs.readFileSync(DB_PATH, 'utf8'))
|
||||
}
|
||||
|
||||
function writeBackup(db) {
|
||||
if (!fs.existsSync(BACKUP_DIR)) fs.mkdirSync(BACKUP_DIR, { recursive: true })
|
||||
const ts = Date.now()
|
||||
const backupPath = path.join(BACKUP_DIR, `deathwatch-bestiary-extracted.json.backup.${ts}.json`)
|
||||
fs.writeFileSync(backupPath, JSON.stringify(db, null, 2), 'utf8')
|
||||
return backupPath
|
||||
}
|
||||
|
||||
function saveDb(db) {
|
||||
db.generatedAt = new Date().toISOString()
|
||||
db.count = db.results.length
|
||||
fs.writeFileSync(DB_PATH, JSON.stringify(db, null, 2), 'utf8')
|
||||
}
|
||||
|
||||
function hasCompleteProfile(entry) {
|
||||
if (!entry || !entry.stats) return false
|
||||
const p = entry.stats.profile
|
||||
if (!p || typeof p !== 'object') return false
|
||||
const required = ['ws','bs','s','t','ag','int','per','wp','fel']
|
||||
for (const k of required) {
|
||||
if (p[k] === undefined || p[k] === null) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
function main() {
|
||||
const db = loadDb()
|
||||
if (!Array.isArray(db.results)) {
|
||||
console.error('DB format unexpected: results is not an array')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
const originalCount = db.results.length
|
||||
const toKeep = []
|
||||
const removed = []
|
||||
|
||||
for (const entry of db.results) {
|
||||
if (hasCompleteProfile(entry)) {
|
||||
toKeep.push(entry)
|
||||
} else {
|
||||
removed.push({ name: entry.bestiaryName || '<unknown>', pdf: entry.pdf || '', page: entry.page || '' })
|
||||
}
|
||||
}
|
||||
|
||||
if (removed.length === 0) {
|
||||
console.log(`No entries removed. DB count remains ${originalCount}`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const backupPath = writeBackup(db)
|
||||
console.log(`Backed up DB to ${backupPath}`)
|
||||
|
||||
db.results = toKeep
|
||||
db.count = toKeep.length
|
||||
db.generatedAt = new Date().toISOString()
|
||||
|
||||
fs.writeFileSync(DB_PATH, JSON.stringify(db, null, 2), 'utf8')
|
||||
|
||||
console.log(`Pruned bestiary: removed ${removed.length} entries (from ${originalCount} -> ${db.count})`)
|
||||
console.log('Removed entries:')
|
||||
for (const r of removed) console.log(` - ${r.name} (pdf=${r.pdf} page=${r.page})`)
|
||||
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
if (require.main === module) main()
|
||||
128
scripts/refine-enemies.js
Normal file
128
scripts/refine-enemies.js
Normal file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env node
|
||||
// Refine and produce a high-confidence enemy DB from structured entries
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const IN_FILE = path.join(__dirname, '..', 'database', 'deathwatch-enemies.json');
|
||||
const OUT_FILE = path.join(__dirname, '..', 'database', 'deathwatch-enemies-highconfidence.json');
|
||||
const OUT_SAMPLE = path.join(__dirname, '..', 'database', 'deathwatch-enemies-highconfidence-sample.json');
|
||||
|
||||
function strongNormalize(s) {
|
||||
if (!s) return '';
|
||||
let t = s.toString();
|
||||
// remove parenthetical notes
|
||||
t = t.replace(/\(.*?\)/g, '');
|
||||
// remove descriptors after dash or colon
|
||||
t = t.split(/[-:—–]/)[0];
|
||||
t = t.replace(/[^\w\s'']/g, ' ');
|
||||
t = t.replace(/\bthe\b\s*/gi, '');
|
||||
// remove common suffix tokens
|
||||
t = t.replace(/\b(elite|swarm|pack|horde|unit|squad|group|team|vehicle|daemon|daemonhost|host|variant)\b/gi, '');
|
||||
t = t.replace(/\s+/g, ' ').trim().toLowerCase();
|
||||
// simple singularize (if very likely plural)
|
||||
if (t.length > 4 && t.endsWith('s') && !t.endsWith('ss')) {
|
||||
t = t.slice(0, -1);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
function mergeNumeric(a,b) {
|
||||
// choose the max non-null
|
||||
if (typeof a !== 'number') return b;
|
||||
if (typeof b !== 'number') return a;
|
||||
return Math.max(a,b);
|
||||
}
|
||||
|
||||
function chooseRepresentative(namesMap) {
|
||||
let bestName = null; let bestCount = -1;
|
||||
for (const [name,count] of Object.entries(namesMap)) {
|
||||
if (count > bestCount) { bestCount = count; bestName = name; }
|
||||
}
|
||||
return bestName || Object.keys(namesMap)[0] || '';
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (!fs.existsSync(IN_FILE)) { console.error('Missing input', IN_FILE); process.exit(1); }
|
||||
const raw = JSON.parse(fs.readFileSync(IN_FILE, 'utf8'));
|
||||
const groups = new Map();
|
||||
|
||||
for (const e of raw) {
|
||||
const name = (e.name || '').trim();
|
||||
if (!name) continue;
|
||||
const key = strongNormalize(name);
|
||||
if (!key) continue;
|
||||
if (!groups.has(key)) {
|
||||
groups.set(key, {
|
||||
key,
|
||||
names: {},
|
||||
count: 0,
|
||||
sources: {},
|
||||
snippets: new Set(),
|
||||
wounds: null,
|
||||
toughness: null,
|
||||
armour: null,
|
||||
armourByLoc: null,
|
||||
pages: new Set()
|
||||
});
|
||||
}
|
||||
const g = groups.get(key);
|
||||
g.count += (e.count||1);
|
||||
g.names[name] = (g.names[name]||0) + (e.count||1);
|
||||
if (e.source) g.sources[e.source] = (g.sources[e.source]||0)+1;
|
||||
for (const s of (e.snippets||[])) {
|
||||
if (s && g.snippets.size < 10) g.snippets.add(s);
|
||||
}
|
||||
if (e.wounds) g.wounds = mergeNumeric(g.wounds, e.wounds);
|
||||
if (e.toughness) g.toughness = mergeNumeric(g.toughness, e.toughness);
|
||||
if (e.armour) g.armour = mergeNumeric(g.armour, e.armour);
|
||||
if (e.armourByLoc && !g.armourByLoc) g.armourByLoc = e.armourByLoc;
|
||||
if (e.page) g.pages.add(e.page);
|
||||
}
|
||||
|
||||
// Score and filter
|
||||
const preferredSources = ['core rulebook','rising tempest','rites of battle','the emperor protects','the emperors chosen','final sanction'];
|
||||
const out = [];
|
||||
for (const g of groups.values()) {
|
||||
let score = g.count;
|
||||
const hasStats = (typeof g.wounds === 'number') || (typeof g.toughness === 'number') || (typeof g.armour === 'number');
|
||||
if (hasStats) score += 50;
|
||||
if (g.pages.size) score += 10;
|
||||
// source boost
|
||||
for (const src of Object.keys(g.sources)) {
|
||||
const lowered = src.toLowerCase();
|
||||
for (const p of preferredSources) if (lowered.includes(p)) score += 5;
|
||||
}
|
||||
// name length penalty for very short keys
|
||||
if (g.key.length < 3) score = 0;
|
||||
|
||||
// filter: require either stats or count>=2 or page hint
|
||||
if (!(hasStats || g.count >= 2 || g.pages.size)) continue;
|
||||
|
||||
const repName = chooseRepresentative(g.names);
|
||||
out.push({
|
||||
id: g.key,
|
||||
name: repName,
|
||||
count: g.count,
|
||||
score,
|
||||
wounds: g.wounds || null,
|
||||
toughness: g.toughness || null,
|
||||
armour: g.armour || null,
|
||||
armourByLoc: g.armourByLoc || null,
|
||||
sources: Object.keys(g.sources),
|
||||
pages: Array.from(g.pages),
|
||||
snippets: Array.from(g.snippets).slice(0,5)
|
||||
});
|
||||
}
|
||||
|
||||
out.sort((a,b) => b.score - a.score || b.count - a.count);
|
||||
// Keep top N high confidence — we'll keep all for now but the user expects ~300; further pruning below
|
||||
// Additional pruning: drop low-score tail: keep only score >= 10 or top 1000
|
||||
const final = out.filter((x,i)=> x.score >= 10 || i < 1000);
|
||||
|
||||
fs.writeFileSync(OUT_FILE, JSON.stringify(final, null, 2));
|
||||
fs.writeFileSync(OUT_SAMPLE, JSON.stringify(final.slice(0,200), null, 2));
|
||||
console.log('Wrote', OUT_FILE, 'with', final.length, 'high-confidence entries (out of', out.length, 'promising groups)');
|
||||
}
|
||||
|
||||
if (require.main === module) main();
|
||||
34
scripts/search-keywords.js
Normal file
34
scripts/search-keywords.js
Normal file
@@ -0,0 +1,34 @@
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const pdf = require('pdf-parse');
|
||||
|
||||
async function search(){
|
||||
const dataDir = path.resolve(__dirname, '../data');
|
||||
const outFile = path.resolve(__dirname, '../database/search-keywords.json');
|
||||
const files = await fs.readdir(dataDir);
|
||||
const pdfs = files.filter(f => f.toLowerCase().endsWith('.pdf'));
|
||||
const keywords = ['alexei','drahj','transpired','transpired alteration','ryza','plasma pistol','custom ryza','xeno-filament','transpired circle'];
|
||||
const results = [];
|
||||
for(const pdfName of pdfs){
|
||||
const filePath = path.join(dataDir, pdfName);
|
||||
try{
|
||||
const data = await fs.readFile(filePath);
|
||||
const parsed = await pdf(data);
|
||||
const text = parsed.text || '';
|
||||
const pages = text.split('\f');
|
||||
for(let i=0;i<pages.length;i++){
|
||||
const pageText = pages[i];
|
||||
const lower = pageText.toLowerCase();
|
||||
for(const kw of keywords){
|
||||
if(lower.includes(kw)){
|
||||
results.push({pdf:pdfName,pageIndex:i,pageNumberApprox:i+1,keyword:kw,snippet: pageText.trim().slice(0,800).replace(/\s+/g,' ')});
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(err){ console.error('err', pdfName, err.message); }
|
||||
}
|
||||
await fs.writeFile(outFile, JSON.stringify({generatedAt: new Date().toISOString(), matches: results}, null, 2));
|
||||
console.log('Wrote', outFile, 'matches:', results.length);
|
||||
}
|
||||
|
||||
search().catch(err=>{ console.error(err); process.exit(2); });
|
||||
40
scripts/search-pdfs-for-name.js
Normal file
40
scripts/search-pdfs-for-name.js
Normal file
@@ -0,0 +1,40 @@
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const pdf = require('pdf-parse');
|
||||
|
||||
async function findName(nameRegex) {
|
||||
const dataDir = path.resolve(__dirname, '../data');
|
||||
const outFile = path.resolve(__dirname, '../database/search-alexei.json');
|
||||
const files = await fs.readdir(dataDir);
|
||||
const pdfs = files.filter(f => f.toLowerCase().endsWith('.pdf'));
|
||||
|
||||
const results = [];
|
||||
for (const pdfName of pdfs) {
|
||||
const filePath = path.join(dataDir, pdfName);
|
||||
try {
|
||||
const dataBuffer = await fs.readFile(filePath);
|
||||
const parsed = await pdf(dataBuffer);
|
||||
const text = parsed.text || '';
|
||||
const pages = text.split('\f');
|
||||
pages.forEach((pageText, idx) => {
|
||||
if (nameRegex.test(pageText)) {
|
||||
results.push({
|
||||
pdf: pdfName,
|
||||
pageIndex: idx,
|
||||
pageNumberApprox: idx + 1,
|
||||
snippet: pageText.trim().slice(0, 800).replace(/\n+/g, ' ')
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('failed', pdfName, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
const out = { generatedAt: new Date().toISOString(), query: nameRegex.toString(), matches: results };
|
||||
await fs.writeFile(outFile, JSON.stringify(out, null, 2));
|
||||
console.log('Wrote', outFile, 'matches:', results.length);
|
||||
}
|
||||
|
||||
const nameRegex = /alexei\s+drahj/i;
|
||||
findName(nameRegex).catch(err => { console.error(err); process.exit(2); });
|
||||
77
scripts/show-bestiary-entry.js
Normal file
77
scripts/show-bestiary-entry.js
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const BESTIARY = path.resolve(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json');
|
||||
|
||||
function usage() {
|
||||
console.log('Usage: node scripts/show-bestiary-entry.js --id <number> | --name "Name"');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const argv = require('minimist')(process.argv.slice(2));
|
||||
if ((!argv.id && !argv.name) || (argv.id && argv.name)) usage();
|
||||
|
||||
let j;
|
||||
try {
|
||||
j = JSON.parse(fs.readFileSync(BESTIARY,'utf8'));
|
||||
} catch (e) {
|
||||
console.error('Failed to load bestiary:', e.message);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const results = j.results || [];
|
||||
let idx = -1;
|
||||
if (argv.id) {
|
||||
const id = Number(argv.id);
|
||||
idx = results.findIndex(r => r.stats && r.stats.id === id || r.id === id || r._id === id);
|
||||
}
|
||||
if (argv.name) {
|
||||
const q = argv.name.toLowerCase();
|
||||
idx = results.findIndex(r => (r.bestiaryName || '').toLowerCase() === q || (r.name||'').toLowerCase() === q);
|
||||
}
|
||||
|
||||
if (idx === -1) {
|
||||
// try fuzzy by includes
|
||||
if (argv.name) {
|
||||
const q = argv.name.toLowerCase();
|
||||
idx = results.findIndex(r => (r.bestiaryName || '').toLowerCase().includes(q) || (r.name||'').toLowerCase().includes(q));
|
||||
}
|
||||
}
|
||||
|
||||
if (idx === -1) {
|
||||
console.error('Entry not found in bestiary for', argv.id || argv.name);
|
||||
console.error('You can list available names via: jq -r ".results[] | .bestiaryName" database/deathwatch-bestiary-extracted.json');
|
||||
process.exit(3);
|
||||
}
|
||||
|
||||
function printEntry(i, label) {
|
||||
const e = results[i];
|
||||
if (!e) return;
|
||||
console.log('\n=== ' + label + ' (index=' + i + ') ===');
|
||||
console.log('bestiaryName:', e.bestiaryName || e.name || '<no-name>');
|
||||
console.log('book:', e.book || e.source || '');
|
||||
console.log('page:', e.page || (e.stats && e.stats.page) || '');
|
||||
console.log('pdf:', e.pdf || '');
|
||||
console.log('snippet:', (e.stats && e.stats.snippet) ? 'present' : (e.snippet ? 'present' : ''));
|
||||
console.log('raw->');
|
||||
console.log(JSON.stringify(e, null, 2));
|
||||
}
|
||||
|
||||
// print previous, current, next
|
||||
if (idx-1 >= 0) printEntry(idx-1, 'Previous');
|
||||
printEntry(idx, 'Current');
|
||||
if (idx+1 < results.length) printEntry(idx+1, 'Next');
|
||||
|
||||
// print a compact stats summary
|
||||
const cur = results[idx];
|
||||
console.log('\n--- Compact stats summary ---');
|
||||
if (cur.stats) {
|
||||
console.log('profile:', cur.stats.profile || cur.stats.stats || null);
|
||||
console.log('movement:', cur.stats.movement || (cur.stats.secondary_stats && cur.stats.secondary_stats.movement) || null);
|
||||
console.log('wounds:', cur.stats.wounds || (cur.stats.secondary_stats && cur.stats.secondary_stats.wounds) || null);
|
||||
console.log('weapons:', cur.stats.weapons || null);
|
||||
console.log('armour:', cur.stats.armour || null);
|
||||
console.log('skills:', cur.stats.skills || null);
|
||||
}
|
||||
console.log('\nDone.');
|
||||
116
scripts/structure-enemies.js
Normal file
116
scripts/structure-enemies.js
Normal file
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env node
|
||||
// Convert enemies-clean.json into structured enemy records with best-effort parsing
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const IN_FILE = path.join(__dirname, '..', 'database', 'enemies-clean.json');
|
||||
const OUT_FILE = path.join(__dirname, '..', 'database', 'deathwatch-enemies.json');
|
||||
const OUT_SAMPLE = path.join(__dirname, '..', 'database', 'deathwatch-enemies-sample.json');
|
||||
|
||||
function parseArmourByLoc(snippet) {
|
||||
const lower = snippet.toLowerCase();
|
||||
const armour = {};
|
||||
// Try common labels
|
||||
const patterns = {
|
||||
head: /head(?:[:\s]+)([0-9]{1,3})/i,
|
||||
body: /body(?:[:\s]+)([0-9]{1,3})/i,
|
||||
ra: /r(?:ight)?\s*arm(?:[:\s]+)([0-9]{1,3})/i,
|
||||
la: /l(?:eft)?\s*arm(?:[:\s]+)([0-9]{1,3})/i,
|
||||
rl: /r(?:ight)?\s*leg(?:[:\s]+)([0-9]{1,3})/i,
|
||||
ll: /l(?:eft)?\s*leg(?:[:\s]+)([0-9]{1,3})/i,
|
||||
ra2: /ra[:\s]*([0-9]{1,3})/i,
|
||||
la2: /la[:\s]*([0-9]{1,3})/i,
|
||||
rl2: /rl[:\s]*([0-9]{1,3})/i,
|
||||
ll2: /ll[:\s]*([0-9]{1,3})/i,
|
||||
ap: /ap[:\s]*([0-9]{1,3})/i
|
||||
};
|
||||
for (const k of ['head','body','ra','la','rl','ll']) {
|
||||
const p = patterns[k] || patterns[k+'2'];
|
||||
if (p) {
|
||||
const m = snippet.match(p);
|
||||
if (m) armour[k] = Number(m[1]);
|
||||
}
|
||||
}
|
||||
// legacy format like "armour: body 10; head 0; ra 8; la 8; rl 8; ll 8"
|
||||
const armorLine = snippet.match(/armou?r[^\n]{0,120}/i);
|
||||
if (armorLine) {
|
||||
const line = armorLine[0];
|
||||
// find numbers after known keys
|
||||
const kvs = line.match(/(body|head|ra|la|rl|ll)[:\s]*([0-9]{1,3})/gi);
|
||||
if (kvs) {
|
||||
for (const kv of kvs) {
|
||||
const m = kv.match(/(body|head|ra|la|rl|ll)[:\s]*([0-9]{1,3})/i);
|
||||
if (m) armour[m[1].toLowerCase()] = Number(m[2]);
|
||||
}
|
||||
}
|
||||
// fallback: single number in armour line may be body AP
|
||||
const single = line.match(/(\d{1,3})/);
|
||||
if (single && !armour.body) armour.body = Number(single[1]);
|
||||
}
|
||||
// If we found any keys, return them, otherwise null
|
||||
return Object.keys(armour).length ? armour : null;
|
||||
}
|
||||
|
||||
function parsePage(snippets, stats) {
|
||||
if (stats && stats.page) return stats.page;
|
||||
for (const s of snippets) {
|
||||
const m = s.match(/page\s*no\.\s*([0-9]{1,4})/i) || s.match(/p(?:age)?\.?\s*([0-9]{1,4})/i);
|
||||
if (m) return Number(m[1]);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isPromising(entry) {
|
||||
const s = entry.stats || {};
|
||||
if (s.wounds || s.toughness || s.ap || s.page) return true;
|
||||
const snippetText = (entry.snippets||[]).join('\n');
|
||||
if (/\barmou?r\b|\bwounds?\b|\bTB\b|\btoughness\b|\bAP\b/i.test(snippetText)) return true;
|
||||
// frequency-based: if seen many times
|
||||
if (entry.count && entry.count >= 2) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function chooseSource(sources) {
|
||||
if (!sources || sources.length === 0) return null;
|
||||
// Prefer Deathwatch core and supplements in order
|
||||
const preferred = ['Deathwatch - Core Rulebook','Deathwatch - Core Rulebook.pdf','Deathwatch - Rising Tempest','Deathwatch - Rites of Battle','Deathwatch - The Emperor Protects','Deathwatch - The Emperors Chosen'];
|
||||
for (const p of preferred) {
|
||||
for (const s of sources) if (s.toLowerCase().includes(p.toLowerCase())) return s;
|
||||
}
|
||||
return sources[0];
|
||||
}
|
||||
|
||||
function main() {
|
||||
if (!fs.existsSync(IN_FILE)) { console.error('Missing input', IN_FILE); process.exit(1); }
|
||||
const raw = JSON.parse(fs.readFileSync(IN_FILE, 'utf8'));
|
||||
const structured = [];
|
||||
for (const e of raw) {
|
||||
if (!isPromising(e)) continue;
|
||||
const name = e.name || '';
|
||||
const stats = e.stats || {};
|
||||
const snippets = e.snippets || [];
|
||||
const armourByLoc = parseArmourByLoc(snippets.join('\n')) || null;
|
||||
const page = parsePage(snippets, stats);
|
||||
const rec = {
|
||||
name: name,
|
||||
wounds: stats.wounds || null,
|
||||
toughness: stats.toughness || null,
|
||||
armour: stats.ap || (armourByLoc && armourByLoc.body) || null,
|
||||
armourByLoc: armourByLoc,
|
||||
source: chooseSource(e.sources || []),
|
||||
page: page,
|
||||
snippets: snippets.slice(0,3),
|
||||
count: e.count || 0
|
||||
};
|
||||
structured.push(rec);
|
||||
}
|
||||
|
||||
// sort by count desc
|
||||
structured.sort((a,b) => (b.count||0) - (a.count||0));
|
||||
fs.writeFileSync(OUT_FILE, JSON.stringify(structured, null, 2));
|
||||
fs.writeFileSync(OUT_SAMPLE, JSON.stringify(structured.slice(0,200), null, 2));
|
||||
console.log('Wrote', OUT_FILE, 'with', structured.length, 'structured promising entries');
|
||||
}
|
||||
|
||||
if (require.main === module) main();
|
||||
137
scripts/upsert-alexei-canonical.js
Normal file
137
scripts/upsert-alexei-canonical.js
Normal file
@@ -0,0 +1,137 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const dataPath = path.resolve(__dirname, '../database/alexei-pdfjs.json');
|
||||
if (!fs.existsSync(dataPath)) {
|
||||
console.error('missing pdfjs findings at', dataPath);
|
||||
process.exit(2);
|
||||
}
|
||||
const j = JSON.parse(fs.readFileSync(dataPath,'utf8'));
|
||||
const findings = j.findings || [];
|
||||
|
||||
// score findings and pick best candidate (prefer exact 'Alexei Drahj' and presence of Movement/Wounds/Profile)
|
||||
function scoreFinding(f){
|
||||
const s = f.text.toLowerCase();
|
||||
let score = 0;
|
||||
if (/alexei\s+drahj/.test(s)) score += 10;
|
||||
if (s.includes('alexei')) score += 4;
|
||||
if (s.includes('movement:')) score += 6;
|
||||
if (s.includes('wounds:')) score += 6;
|
||||
if (s.includes('profile')) score += 4;
|
||||
if (s.includes('weapons:')) score += 2;
|
||||
return score;
|
||||
}
|
||||
|
||||
let best = null;
|
||||
for (const f of findings){
|
||||
if (!best || scoreFinding(f) > scoreFinding(best)) best = f;
|
||||
}
|
||||
if (!best){ console.error('no alexei finding'); process.exit(1); }
|
||||
|
||||
const txt = best.text;
|
||||
// narrow to the Alexei subsection
|
||||
let pos = txt.search(/alexei\s+drahj/i);
|
||||
if (pos < 0) pos = txt.search(/alexei/i);
|
||||
const windowText = txt.substr(Math.max(0, pos-500), 2500);
|
||||
|
||||
function extractProfile(s){
|
||||
// Handle optional leading parenthetical modifiers like "(10) (12) (8)" followed by the main 9-value profile row.
|
||||
const modifiersMatch = s.match(/^(?:[^\n]{0,120})?\(?\s*(\d{1,2})\s*\)?(?:\s*\(?\s*(\d{1,2})\s*\)\s*)?(?:\(?\s*(\d{1,2})\s*\)\s*)?/m);
|
||||
let modifiers = null;
|
||||
if (modifiersMatch && (modifiersMatch[1] || modifiersMatch[2] || modifiersMatch[3])) {
|
||||
modifiers = [modifiersMatch[1], modifiersMatch[2], modifiersMatch[3]].filter(Boolean).map(n=>parseInt(n,10));
|
||||
}
|
||||
// Find all runs of 9 numbers and pick the last (or the one after modifiers)
|
||||
const seqAll = Array.from(s.matchAll(/(?:\b\d{1,2}\b(?:[^\d\n]{1,6}\d{1,2}\b){8})/gm));
|
||||
if (!seqAll || seqAll.length===0) return null;
|
||||
let chosenSeq = null;
|
||||
if (modifiers) {
|
||||
for (const m of seqAll) {
|
||||
if (m.index > (modifiersMatch.index || 0)) { chosenSeq = m[0]; break; }
|
||||
}
|
||||
}
|
||||
if (!chosenSeq) chosenSeq = seqAll[seqAll.length-1][0];
|
||||
const nums = chosenSeq.replace(/[^0-9\s]/g,' ').trim().split(/\s+/).map(n=>parseInt(n,10));
|
||||
if (nums.length<9) return null;
|
||||
const keys = ['ws','bs','s','t','ag','int','per','wp','fel'];
|
||||
const obj = {};
|
||||
keys.forEach((k,i)=>obj[k]=nums[i]);
|
||||
if (modifiers) obj.modifiers = modifiers;
|
||||
return obj;
|
||||
}
|
||||
|
||||
function extractMovement(s){ const m = s.match(/Movement:\s*([0-9\/]+(?:\/[0-9]+)*)/i); return m?m[1].trim():null; }
|
||||
function extractWounds(s){ const m = s.match(/Wounds:\s*(\d{1,3})/i); return m?parseInt(m[1],10):null; }
|
||||
|
||||
function extractBlock(s, startLabel, endLabels){
|
||||
const reStart = new RegExp(startLabel+'\s*','i');
|
||||
const si = s.search(reStart);
|
||||
if (si<0) return null;
|
||||
const rest = s.substr(si+startLabel.length);
|
||||
// find earliest occurrence of any endLabel
|
||||
let endPos = rest.length;
|
||||
for (const el of endLabels){
|
||||
const r = new RegExp('\\b'+el+'\\b','i');
|
||||
const m = rest.search(r);
|
||||
if (m>=0 && m<endPos) endPos = m;
|
||||
}
|
||||
const blk = rest.substr(0,endPos).trim();
|
||||
return blk.replace(/\s+/g,' ').replace(/^[,:\s]+/,'').trim();
|
||||
}
|
||||
|
||||
const profile = extractProfile(windowText);
|
||||
const movement = extractMovement(windowText);
|
||||
const wounds = extractWounds(windowText);
|
||||
const skills = extractBlock(windowText,'Skills:', ['Talents:','Traits:','Armour:','Weapons:','Gear:']) || null;
|
||||
const talents = extractBlock(windowText,'Talents:', ['Traits:','Armour:','Weapons:','Gear:']) || null;
|
||||
const traits = extractBlock(windowText,'Traits:', ['Armour:','Weapons:','Gear:','Transpired','†']) || extractBlock(windowText,'Traits:', ['Armour:','Weapons:','Gear:']) || null;
|
||||
const armour = extractBlock(windowText,'Armour:', ['Weapons:','Gear:']) || null;
|
||||
const weapons = extractBlock(windowText,'Weapons:', ['Gear:','†']) || null;
|
||||
const gear = extractBlock(windowText,'Gear:', ['†']) || null;
|
||||
|
||||
const canonical = {
|
||||
name: 'Alexei Drahj',
|
||||
source: best.pdf || 'Deathwatch - The Emperor Protects.pdf',
|
||||
pdfPage: best.page || null,
|
||||
printedPage: (best.page && Number.isInteger(best.page)) ? best.page - 1 : null,
|
||||
stats: {
|
||||
profile,
|
||||
movement,
|
||||
wounds,
|
||||
skills,
|
||||
talents,
|
||||
traits,
|
||||
armour,
|
||||
weapons,
|
||||
gear,
|
||||
snippet: windowText.replace(/\n+/g,' ').trim().slice(0,2000)
|
||||
},
|
||||
insertedAt: new Date().toISOString()
|
||||
};
|
||||
|
||||
// write alexei-canonical.json
|
||||
const outPath = path.resolve(__dirname, '../database/alexei-canonical.json');
|
||||
fs.writeFileSync(outPath, JSON.stringify(canonical, null, 2), 'utf8');
|
||||
console.log('Wrote', outPath);
|
||||
|
||||
// upsert into deathwatch-bestiary-extracted.json
|
||||
const bestiaryPath = path.resolve(__dirname, '../database/deathwatch-bestiary-extracted.json');
|
||||
let bestiary = { results: [], count: 0 };
|
||||
if (fs.existsSync(bestiaryPath)){
|
||||
try{ bestiary = JSON.parse(fs.readFileSync(bestiaryPath,'utf8')); }catch(e){ console.error('failed parse bestiary', e.message); }
|
||||
}
|
||||
// ensure structure
|
||||
if (!Array.isArray(bestiary.results)) bestiary.results = [];
|
||||
|
||||
// find existing Alexei entry by name
|
||||
const idx = bestiary.results.findIndex(r => r.name && /alexei\s+drahj/i.test(r.name));
|
||||
const entry = { name: canonical.name, source: canonical.source, page: canonical.printedPage, pdfPage: canonical.pdfPage, stats: canonical.stats };
|
||||
if (idx>=0){ bestiary.results[idx] = entry; } else { bestiary.results.push(entry); }
|
||||
bestiary.count = bestiary.results.length;
|
||||
fs.writeFileSync(bestiaryPath, JSON.stringify(bestiary, null, 2), 'utf8');
|
||||
console.log('Upserted Alexei into', bestiaryPath);
|
||||
|
||||
console.log('Canonical record:');
|
||||
console.log(JSON.stringify(canonical.stats, null, 2));
|
||||
|
||||
process.exit(0);
|
||||
106
scripts/upsert-allewis-from-5001.js
Normal file
106
scripts/upsert-allewis-from-5001.js
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
async function main(){
|
||||
const apiUrl = 'http://192.168.1.144:5001/api/search?q=Allewis';
|
||||
console.log('Fetching:', apiUrl);
|
||||
let res;
|
||||
try{
|
||||
res = await fetch(apiUrl);
|
||||
}catch(e){
|
||||
console.error('Fetch failed:', e.message);
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
if(!res.ok){
|
||||
console.error('API returned', res.status, res.statusText);
|
||||
process.exitCode = 3;
|
||||
return;
|
||||
}
|
||||
const body = await res.json();
|
||||
const stat = (body && body.statblocks && body.statblocks[0]) || null;
|
||||
if(!stat){
|
||||
console.error('No statblock found in API response');
|
||||
process.exitCode = 4;
|
||||
return;
|
||||
}
|
||||
|
||||
const movement = stat.secondary_stats && stat.secondary_stats.movement;
|
||||
const wounds = stat.secondary_stats && stat.secondary_stats.wounds;
|
||||
console.log('Found statblock:', stat.name, 'page', stat.page, 'movement', movement, 'wounds', wounds);
|
||||
|
||||
const bestiaryPath = path.resolve(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json');
|
||||
if(!fs.existsSync(bestiaryPath)){
|
||||
console.error('Bestiary file not found:', bestiaryPath);
|
||||
process.exitCode = 5;
|
||||
return;
|
||||
}
|
||||
const raw = fs.readFileSync(bestiaryPath, 'utf8');
|
||||
// backup
|
||||
const backupsDir = path.resolve(__dirname, '..', 'database', 'backups');
|
||||
if(!fs.existsSync(backupsDir)) fs.mkdirSync(backupsDir, { recursive: true });
|
||||
const backupPath = path.join(backupsDir, 'deathwatch-bestiary-extracted.json.backup.' + Date.now() + '.json');
|
||||
fs.writeFileSync(backupPath, raw, 'utf8');
|
||||
console.log('Wrote backup:', backupPath);
|
||||
|
||||
let obj;
|
||||
try{ obj = JSON.parse(raw); } catch(e){
|
||||
console.error('Failed to parse bestiary JSON:', e.message);
|
||||
process.exitCode = 6; return;
|
||||
}
|
||||
|
||||
const nameToFind = (stat.name || 'Allewis').toLowerCase();
|
||||
const pageToFind = String(stat.page || '89');
|
||||
|
||||
let idx = obj.results.findIndex(r => {
|
||||
if(!r) return false;
|
||||
const bn = (r.bestiaryName || '').toLowerCase();
|
||||
if(bn && bn.includes(nameToFind)) return true;
|
||||
if(r.page && String(r.page) === pageToFind) return true;
|
||||
// fallback check snippet
|
||||
if(r.stats && r.stats.snippet && String(r.stats.snippet).toLowerCase().includes(nameToFind)) return true;
|
||||
return false;
|
||||
});
|
||||
|
||||
if(idx === -1){
|
||||
console.log('No matching entry found — will append a minimal entry.');
|
||||
const newEntry = {
|
||||
bestiaryName: stat.name || 'Prince-Prefect Allewis',
|
||||
book: stat.source || 'Unknown',
|
||||
page: String(stat.page || '89'),
|
||||
pdf: '',
|
||||
pageText: '',
|
||||
stats: {
|
||||
profile: null,
|
||||
movement: movement || null,
|
||||
wounds: (typeof wounds === 'number') ? wounds : (wounds ? Number(wounds) : null),
|
||||
toughness: null,
|
||||
skills: null,
|
||||
talents: null,
|
||||
traits: null,
|
||||
armour: null,
|
||||
weapons: null,
|
||||
gear: null,
|
||||
snippet: ''
|
||||
}
|
||||
};
|
||||
obj.results.push(newEntry);
|
||||
console.log('Appended new entry for', newEntry.bestiaryName);
|
||||
}else{
|
||||
console.log('Found existing entry at index', idx, 'name=', obj.results[idx].bestiaryName);
|
||||
const target = obj.results[idx];
|
||||
if(!target.stats) target.stats = {};
|
||||
target.stats.movement = movement || target.stats.movement || null;
|
||||
target.stats.wounds = (typeof wounds === 'number') ? wounds : (wounds ? Number(wounds) : target.stats.wounds || null);
|
||||
console.log('Updated movement/wounds on existing entry.');
|
||||
}
|
||||
|
||||
fs.writeFileSync(bestiaryPath, JSON.stringify(obj, null, 2), 'utf8');
|
||||
console.log('Wrote updated bestiary to', bestiaryPath);
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error('Unhandled error', e);
|
||||
process.exitCode = 99;
|
||||
});
|
||||
131
scripts/upsert-missing-movement-from-5001.js
Normal file
131
scripts/upsert-missing-movement-from-5001.js
Normal file
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const util = require('util');
|
||||
|
||||
async function fetchJson(url){
|
||||
try{
|
||||
const res = await fetch(url);
|
||||
if(!res.ok) throw new Error(`${res.status} ${res.statusText}`);
|
||||
return await res.json();
|
||||
}catch(e){
|
||||
console.error('Fetch error for', url, e.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function makeBackup(bestiaryPath){
|
||||
const raw = fs.readFileSync(bestiaryPath, 'utf8');
|
||||
const backupsDir = path.resolve(__dirname, '..', 'database', 'backups');
|
||||
if(!fs.existsSync(backupsDir)) fs.mkdirSync(backupsDir, { recursive: true });
|
||||
const backupPath = path.join(backupsDir, 'deathwatch-bestiary-extracted.json.backup.' + Date.now() + '.json');
|
||||
fs.writeFileSync(backupPath, raw, 'utf8');
|
||||
return backupPath;
|
||||
}
|
||||
|
||||
function normalizeName(s){
|
||||
if(!s) return '';
|
||||
return String(s).replace(/[^a-z0-9\s]/gi,' ').replace(/\s+/g,' ').trim();
|
||||
}
|
||||
|
||||
async function main(){
|
||||
const bestiaryPath = path.resolve(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json');
|
||||
if(!fs.existsSync(bestiaryPath)){ console.error('Bestiary not found at', bestiaryPath); process.exit(1); }
|
||||
let obj;
|
||||
try{ obj = JSON.parse(fs.readFileSync(bestiaryPath, 'utf8')); }catch(e){ console.error('Failed to parse bestiary:', e.message); process.exit(2); }
|
||||
|
||||
const entries = obj.results || [];
|
||||
const need = [];
|
||||
for(const e of entries){
|
||||
if(!e || !e.stats) continue;
|
||||
const mv = e.stats.movement;
|
||||
const wd = e.stats.wounds;
|
||||
if(!mv || mv === null || mv === '\\n' || String(mv).trim() === '') need.push(e);
|
||||
// also consider if wounds missing
|
||||
else if(wd === null || wd === undefined) need.push(e);
|
||||
}
|
||||
|
||||
console.log('Total entries:', entries.length, 'Missing movement/wounds count:', need.length);
|
||||
if(need.length === 0){ console.log('Nothing to do.'); return; }
|
||||
|
||||
const backupPath = makeBackup(bestiaryPath);
|
||||
console.log('Backup created:', backupPath);
|
||||
|
||||
const updated = [];
|
||||
const appended = [];
|
||||
|
||||
for(const e of need){
|
||||
const name = e.bestiaryName || (e.stats && e.stats.snippet && e.stats.snippet.split('\n')[0]) || e.apiName || '';
|
||||
const q = encodeURIComponent((name || '').trim() || '');
|
||||
if(!q){ console.log('Skipping entry with no name'); continue; }
|
||||
const url = `http://192.168.1.144:5001/api/search?q=${q}`;
|
||||
console.log('\nQuerying API for', name, '->', url);
|
||||
const body = await fetchJson(url);
|
||||
if(!body || !body.statblocks || body.statblocks.length === 0){
|
||||
console.log('No statblocks returned for', name);
|
||||
continue;
|
||||
}
|
||||
// try to pick best statblock: prefer matching page or exact name
|
||||
const candidates = body.statblocks;
|
||||
let pick = candidates[0];
|
||||
// attempt better match
|
||||
for(const s of candidates){
|
||||
if(e.page && s.page && String(e.page) === String(s.page)){ pick = s; break; }
|
||||
const bn = normalizeName(e.bestiaryName || '');
|
||||
const sn = normalizeName(s.name || '');
|
||||
if(bn && sn && sn.includes(bn)) { pick = s; break; }
|
||||
}
|
||||
|
||||
const movement = pick.secondary_stats && pick.secondary_stats.movement;
|
||||
const wounds = pick.secondary_stats && pick.secondary_stats.wounds;
|
||||
if(!movement && (wounds === null || wounds === undefined)){
|
||||
console.log('API candidate has no movement/wounds for', name);
|
||||
continue;
|
||||
}
|
||||
|
||||
// find index in entries again (safe)
|
||||
const idx = entries.findIndex(r => r === e);
|
||||
if(idx === -1){
|
||||
// append
|
||||
const newEntry = {
|
||||
bestiaryName: pick.name || name,
|
||||
book: pick.source || 'Unknown',
|
||||
page: String(pick.page || ''),
|
||||
pdf: '',
|
||||
pageText: '',
|
||||
stats: {
|
||||
profile: null,
|
||||
movement: movement || null,
|
||||
wounds: (typeof wounds === 'number') ? wounds : (wounds ? Number(wounds) : null),
|
||||
toughness: null,
|
||||
skills: null,
|
||||
talents: null,
|
||||
traits: null,
|
||||
armour: null,
|
||||
weapons: null,
|
||||
gear: null,
|
||||
snippet: ''
|
||||
}
|
||||
};
|
||||
entries.push(newEntry);
|
||||
appended.push(newEntry.bestiaryName);
|
||||
console.log('Appended new entry for', newEntry.bestiaryName);
|
||||
}else{
|
||||
const target = entries[idx];
|
||||
const before = { movement: target.stats.movement, wounds: target.stats.wounds };
|
||||
if(movement) target.stats.movement = movement;
|
||||
if(wounds !== null && wounds !== undefined) target.stats.wounds = (typeof wounds === 'number') ? wounds : (wounds ? Number(wounds) : target.stats.wounds);
|
||||
updated.push({ name: target.bestiaryName || name, before, after: { movement: target.stats.movement, wounds: target.stats.wounds } });
|
||||
console.log('Updated:', target.bestiaryName, '-> movement:', target.stats.movement, 'wounds:', target.stats.wounds);
|
||||
}
|
||||
}
|
||||
|
||||
// write back
|
||||
obj.results = entries;
|
||||
fs.writeFileSync(bestiaryPath, JSON.stringify(obj, null, 2), 'utf8');
|
||||
console.log('\nFinished. Updated:', updated.length, 'Appended:', appended.length);
|
||||
if(updated.length) console.log('Updated details:', util.inspect(updated, { depth: 3 }));
|
||||
if(appended.length) console.log('Appended names:', appended.join(', '));
|
||||
}
|
||||
|
||||
main().catch(e => { console.error('Unhandled', e); process.exit(99); });
|
||||
93
scripts/upsert-missing-movement-from-pdfs.js
Normal file
93
scripts/upsert-missing-movement-from-pdfs.js
Normal file
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
function makeBackup(bestiaryPath){
|
||||
const raw = fs.readFileSync(bestiaryPath, 'utf8');
|
||||
const backupsDir = path.resolve(__dirname, '..', 'database', 'backups');
|
||||
if(!fs.existsSync(backupsDir)) fs.mkdirSync(backupsDir, { recursive: true });
|
||||
const backupPath = path.join(backupsDir, 'deathwatch-bestiary-extracted.json.backup.pdffallback.' + Date.now() + '.json');
|
||||
fs.writeFileSync(backupPath, raw, 'utf8');
|
||||
return backupPath;
|
||||
}
|
||||
|
||||
function extractFromText(text){
|
||||
if(!text) return {};
|
||||
const mvMatch = text.match(/Movement:\s*([0-9\s\/\-]+)\b/i);
|
||||
const wdMatch = text.match(/Wounds:\s*([0-9]+)/i);
|
||||
return {
|
||||
movement: mvMatch ? mvMatch[1].trim().replace(/\s+/g,' ') : null,
|
||||
wounds: wdMatch ? Number(wdMatch[1]) : null
|
||||
};
|
||||
}
|
||||
|
||||
function scanDatabaseFilesForName(name){
|
||||
const dbDir = path.resolve(__dirname, '..', 'database');
|
||||
const files = fs.readdirSync(dbDir).filter(f => f.endsWith('.json'));
|
||||
const results = [];
|
||||
const lower = (name || '').toLowerCase();
|
||||
for(const f of files){
|
||||
const p = path.join(dbDir, f);
|
||||
let raw;
|
||||
try{ raw = fs.readFileSync(p, 'utf8'); }catch(e){ continue; }
|
||||
if(raw.toLowerCase().includes(lower)){
|
||||
// try to extract
|
||||
const ex = extractFromText(raw);
|
||||
if(ex.movement || ex.wounds) results.push({ file: f, movement: ex.movement, wounds: ex.wounds, snippet: raw.substr(Math.max(0, raw.toLowerCase().indexOf(lower)-200), 600) });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
function normalizeName(s){ return (s||'').replace(/[^a-z0-9\s]/gi,' ').replace(/\s+/g,' ').trim(); }
|
||||
|
||||
async function main(){
|
||||
const bestiaryPath = path.resolve(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json');
|
||||
if(!fs.existsSync(bestiaryPath)){ console.error('Bestiary not found:', bestiaryPath); process.exit(1); }
|
||||
const obj = JSON.parse(fs.readFileSync(bestiaryPath, 'utf8'));
|
||||
const entries = obj.results || [];
|
||||
const need = entries.filter(e => !(e && e.stats && e.stats.movement && String(e.stats.movement).trim() !== ''));
|
||||
console.log('Entries needing movement/wounds:', need.length);
|
||||
if(need.length === 0){ console.log('Nothing to do'); return; }
|
||||
|
||||
const backup = makeBackup(bestiaryPath);
|
||||
console.log('Backup created:', backup);
|
||||
|
||||
const updated = [];
|
||||
|
||||
for(const e of need){
|
||||
const name = e.bestiaryName || (e.stats && e.stats.snippet) || '';
|
||||
if(!name){ console.log('Skipping nameless entry'); continue; }
|
||||
console.log('\nScanning database JSONs for', name);
|
||||
// try relaxed searches: exact name, normalized name token, first two words
|
||||
const candidates = [];
|
||||
const direct = scanDatabaseFilesForName(name);
|
||||
candidates.push(...direct);
|
||||
if(candidates.length === 0){
|
||||
const nrm = normalizeName(name);
|
||||
if(nrm){
|
||||
const tokens = nrm.split(' ').slice(0,3).join(' ');
|
||||
const alt = scanDatabaseFilesForName(tokens);
|
||||
candidates.push(...alt);
|
||||
}
|
||||
}
|
||||
if(candidates.length === 0){ console.log('No PDF-derived candidates for', name); continue; }
|
||||
console.log('Found candidates from files:', candidates.map(c=>c.file).join(', '));
|
||||
// pick first with movement or wounds
|
||||
let picked = null;
|
||||
for(const c of candidates){ if(c.movement || c.wounds){ picked = c; break; } }
|
||||
if(!picked) picked = candidates[0];
|
||||
const before = { movement: e.stats && e.stats.movement, wounds: e.stats && e.stats.wounds };
|
||||
if(!e.stats) e.stats = {};
|
||||
if(picked.movement) e.stats.movement = picked.movement;
|
||||
if(picked.wounds !== null && picked.wounds !== undefined) e.stats.wounds = picked.wounds;
|
||||
updated.push({ name: e.bestiaryName, file: picked.file, before, after: { movement: e.stats.movement, wounds: e.stats.wounds } });
|
||||
console.log('Updated', e.bestiaryName, '->', e.stats.movement, e.stats.wounds);
|
||||
}
|
||||
|
||||
fs.writeFileSync(bestiaryPath, JSON.stringify(obj, null, 2), 'utf8');
|
||||
console.log('\nDone. Updated', updated.length, 'entries.');
|
||||
if(updated.length) console.log('Details:', updated);
|
||||
}
|
||||
|
||||
main().catch(e => { console.error('Fatal', e); process.exit(99); });
|
||||
93
scripts/upsert-missing-profiles-from-pdfs.js
Normal file
93
scripts/upsert-missing-profiles-from-pdfs.js
Normal file
@@ -0,0 +1,93 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const BESTIARY = path.resolve(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json');
|
||||
const DB_DIR = path.resolve(__dirname, '..', 'database');
|
||||
const BACKUPS = path.resolve(__dirname, '..', 'database', 'backups');
|
||||
|
||||
function backup(srcPath, tag) {
|
||||
if (!fs.existsSync(BACKUPS)) fs.mkdirSync(BACKUPS, { recursive: true });
|
||||
const ts = Date.now();
|
||||
const dest = path.join(BACKUPS, path.basename(srcPath) + `.backup.${tag}.${ts}.json`);
|
||||
fs.copyFileSync(srcPath, dest);
|
||||
return dest;
|
||||
}
|
||||
|
||||
function loadJSON(p) {
|
||||
try { return JSON.parse(fs.readFileSync(p,'utf8')); } catch(e){ return null; }
|
||||
}
|
||||
|
||||
function findCandidates(nameLower) {
|
||||
const files = fs.readdirSync(DB_DIR).filter(f => f.endsWith('.json'));
|
||||
const candidates = [];
|
||||
for (const f of files) {
|
||||
const p = path.join(DB_DIR, f);
|
||||
let j;
|
||||
try { j = JSON.parse(fs.readFileSync(p,'utf8')); } catch(e){ continue; }
|
||||
if (!j.results || !Array.isArray(j.results)) continue;
|
||||
for (const r of j.results) {
|
||||
if (!r.stats) continue;
|
||||
if (r.stats.profile && Object.keys(r.stats.profile).length > 0) {
|
||||
// match by snippet or pageText or pdf filename
|
||||
const hay = (r.stats.snippet || r.pageText || r.snippet || '').toLowerCase();
|
||||
if (hay.includes(nameLower)) {
|
||||
candidates.push({file: p, result: r});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const bestiary = loadJSON(BESTIARY);
|
||||
if (!bestiary) { console.error('Failed loading bestiary'); process.exit(2); }
|
||||
const results = bestiary.results || [];
|
||||
const toUpdate = [];
|
||||
|
||||
for (const entry of results) {
|
||||
const name = entry.bestiaryName || '';
|
||||
const stats = entry.stats || {};
|
||||
const needsProfile = !stats.profile || Object.values(stats.profile || {}).every(v => v === null || v === undefined);
|
||||
if (!needsProfile) continue;
|
||||
const nameLower = name.toLowerCase();
|
||||
const candidates = findCandidates(nameLower);
|
||||
if (candidates.length) {
|
||||
// pick first candidate
|
||||
const c = candidates[0];
|
||||
toUpdate.push({entry, candidate: c});
|
||||
}
|
||||
}
|
||||
|
||||
if (!toUpdate.length) { console.log('No profile candidates found in local PDF JSONs.'); process.exit(0); }
|
||||
|
||||
console.log(`Found ${toUpdate.length} candidate(s) to upsert.`);
|
||||
backup(BESTIARY, 'pdffill');
|
||||
|
||||
let updated = 0;
|
||||
for (const u of toUpdate) {
|
||||
const e = u.entry;
|
||||
const r = u.candidate.result;
|
||||
e.pdf = path.basename(u.candidate.file);
|
||||
e.page = r.foundPage || e.page || null;
|
||||
e.pageText = e.pageText || null;
|
||||
e.stats = e.stats || {};
|
||||
if (r.stats.profile) e.stats.profile = r.stats.profile;
|
||||
if (r.stats.movement) e.stats.movement = r.stats.movement;
|
||||
if (r.stats.wounds) e.stats.wounds = r.stats.wounds;
|
||||
if (r.stats.skills) e.stats.skills = r.stats.skills;
|
||||
if (r.stats.talents) e.stats.talents = r.stats.talents;
|
||||
if (r.stats.traits) e.stats.traits = r.stats.traits;
|
||||
if (r.stats.armour) e.stats.armour = r.stats.armour;
|
||||
if (r.stats.weapons) e.stats.weapons = r.stats.weapons;
|
||||
if (r.stats.gear) e.stats.gear = r.stats.gear;
|
||||
if (r.stats.snippet) e.stats.snippet = (e.stats.snippet || '') + '\n' + r.stats.snippet;
|
||||
updated++;
|
||||
console.log(`Upserted profile for: ${e.bestiaryName} from ${u.candidate.file}`);
|
||||
}
|
||||
|
||||
fs.writeFileSync(BESTIARY, JSON.stringify(bestiary, null, 2), 'utf8');
|
||||
console.log(`Wrote ${updated} updates to ${BESTIARY}`);
|
||||
}
|
||||
|
||||
main();
|
||||
77
scripts/upsert-missing-wounds-from-pdfs.js
Normal file
77
scripts/upsert-missing-wounds-from-pdfs.js
Normal file
@@ -0,0 +1,77 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const BESTIARY = path.resolve(__dirname, '..', 'database', 'deathwatch-bestiary-extracted.json');
|
||||
const DB_DIR = path.resolve(__dirname, '..', 'database');
|
||||
const BACKUPS = path.resolve(__dirname, '..', 'database', 'backups');
|
||||
|
||||
function backup(srcPath, tag) {
|
||||
if (!fs.existsSync(BACKUPS)) fs.mkdirSync(BACKUPS, { recursive: true });
|
||||
const ts = Date.now();
|
||||
const dest = path.join(BACKUPS, path.basename(srcPath) + `.backup.${tag}.${ts}.json`);
|
||||
fs.copyFileSync(srcPath, dest);
|
||||
return dest;
|
||||
}
|
||||
|
||||
function loadJSON(p) {
|
||||
try { return JSON.parse(fs.readFileSync(p,'utf8')); } catch(e){ return null; }
|
||||
}
|
||||
|
||||
function findWoundsFor(nameLower) {
|
||||
const files = fs.readdirSync(DB_DIR).filter(f => f.endsWith('.json'));
|
||||
for (const f of files) {
|
||||
const p = path.join(DB_DIR, f);
|
||||
let j;
|
||||
try { j = JSON.parse(fs.readFileSync(p,'utf8')); } catch(e){ continue; }
|
||||
if (!j.results || !Array.isArray(j.results)) continue;
|
||||
for (const r of j.results) {
|
||||
if (!r.stats) continue;
|
||||
if (r.stats.wounds) {
|
||||
const hay = (r.stats.snippet || r.pageText || r.snippet || '').toLowerCase();
|
||||
if (hay.includes(nameLower)) return {file: p, result: r};
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const bestiary = loadJSON(BESTIARY);
|
||||
if (!bestiary) { console.error('Failed loading bestiary'); process.exit(2); }
|
||||
const results = bestiary.results || [];
|
||||
const toUpdate = [];
|
||||
|
||||
for (const entry of results) {
|
||||
const stats = entry.stats || {};
|
||||
const woundsMissing = stats.wounds === null || stats.wounds === undefined;
|
||||
if (!woundsMissing) continue;
|
||||
const nameLower = (entry.bestiaryName || '').toLowerCase();
|
||||
const found = findWoundsFor(nameLower);
|
||||
if (found) toUpdate.push({entry, found});
|
||||
}
|
||||
|
||||
if (!toUpdate.length) { console.log('No wounds candidates found in local PDF JSONs.'); process.exit(0); }
|
||||
|
||||
console.log(`Found ${toUpdate.length} wounds candidate(s) to upsert.`);
|
||||
backup(BESTIARY, 'pdffill-wounds');
|
||||
|
||||
let updated = 0;
|
||||
for (const u of toUpdate) {
|
||||
const e = u.entry;
|
||||
const r = u.found.result;
|
||||
e.pdf = path.basename(u.found.file);
|
||||
e.page = r.foundPage || e.page || null;
|
||||
e.pageText = e.pageText || null;
|
||||
e.stats = e.stats || {};
|
||||
if (r.stats.wounds) {
|
||||
e.stats.wounds = r.stats.wounds;
|
||||
updated++;
|
||||
console.log(`Set wounds for ${e.bestiaryName} -> ${r.stats.wounds} (from ${u.found.file})`);
|
||||
}
|
||||
}
|
||||
|
||||
fs.writeFileSync(BESTIARY, JSON.stringify(bestiary, null, 2), 'utf8');
|
||||
console.log(`Wrote ${updated} wounds updates to ${BESTIARY}`);
|
||||
}
|
||||
|
||||
main();
|
||||
0
scripts/upsert-remote-statblocks.js
Normal file
0
scripts/upsert-remote-statblocks.js
Normal file
94
scripts/validate-bestiary.js
Normal file
94
scripts/validate-bestiary.js
Normal file
@@ -0,0 +1,94 @@
|
||||
// scripts/validate-bestiary.js
|
||||
// Validate structure of database/deathwatch-bestiary-extracted.json
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const BESTIARY_PATH = path.join(__dirname, '../database/deathwatch-bestiary-extracted.json');
|
||||
|
||||
function isInt(n){return Number.isInteger(n);}
|
||||
function isNonEmptyString(s){return typeof s === 'string' && s.trim().length>0}
|
||||
|
||||
function validateProfile(profile){
|
||||
const keys = ['ws','bs','s','t','ag','int','per','wp','fel'];
|
||||
if(!profile || typeof profile !== 'object') return {ok:false,errs:['missing profile object']};
|
||||
const errs = [];
|
||||
for(const k of keys){
|
||||
const v = profile[k];
|
||||
if(v===null || v===undefined) errs.push(`profile.${k} is missing`);
|
||||
else if(!isInt(v)) errs.push(`profile.${k} is not integer (${v})`);
|
||||
}
|
||||
return {ok:errs.length===0, errs};
|
||||
}
|
||||
|
||||
function validateMovement(mv){
|
||||
if(mv===null || mv===undefined) return {ok:false,errs:['movement missing']};
|
||||
if(typeof mv === 'string' && mv.match(/^\d+\/\d+\/\d+\/\d+$/)) return {ok:true,errs:[]};
|
||||
return {ok:false,errs:[`movement has unexpected format: ${mv}`]};
|
||||
}
|
||||
|
||||
function validateWounds(w){
|
||||
if(w===null || w===undefined) return {ok:false,errs:['wounds missing']};
|
||||
if(typeof w === 'number' && Number.isFinite(w)) return {ok:true,errs:[]};
|
||||
if(typeof w === 'string' && /^\d+$/.test(w)) return {ok:true,errs:[]};
|
||||
return {ok:false,errs:[`wounds invalid: ${w}`]};
|
||||
}
|
||||
|
||||
function validateEntry(e){
|
||||
const errs = [];
|
||||
if(!isNonEmptyString(e.bestiaryName) && !isNonEmptyString(e.name)) errs.push('missing bestiaryName/name');
|
||||
if(!isNonEmptyString(e.source) && !isNonEmptyString(e.book)) errs.push('missing source/book');
|
||||
|
||||
const stats = e.stats || {};
|
||||
const profile = stats.profile || {};
|
||||
const p = validateProfile(profile);
|
||||
if(!p.ok) errs.push(...p.errs);
|
||||
|
||||
const mv = validateMovement(stats.movement);
|
||||
if(!mv.ok) errs.push(...mv.errs);
|
||||
|
||||
const w = validateWounds(stats.wounds);
|
||||
if(!w.ok) errs.push(...w.errs);
|
||||
|
||||
// weapons or gear recommended
|
||||
if(!stats.weapons && !stats.gear) errs.push('neither weapons nor gear present');
|
||||
|
||||
return errs;
|
||||
}
|
||||
|
||||
function run(){
|
||||
if(!fs.existsSync(BESTIARY_PATH)){
|
||||
console.error('Bestiary file not found at', BESTIARY_PATH);
|
||||
process.exit(2);
|
||||
}
|
||||
let obj;
|
||||
try{ obj = JSON.parse(fs.readFileSync(BESTIARY_PATH,'utf8')); } catch(e){ console.error('Failed to parse bestiary JSON:', e.message); process.exit(2); }
|
||||
const arr = Array.isArray(obj) ? obj : (Array.isArray(obj.results) ? obj.results : []);
|
||||
if(!Array.isArray(arr)){
|
||||
console.error('Bestiary format unrecognized');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
let failures=0;
|
||||
const report = [];
|
||||
arr.forEach((entry, idx)=>{
|
||||
const errs = validateEntry(entry);
|
||||
if(errs.length){
|
||||
failures++;
|
||||
report.push({index: idx, name: entry.bestiaryName || entry.name || '(unknown)', errors: errs});
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`Entries checked: ${arr.length}. Failures: ${failures}`);
|
||||
if(report.length){
|
||||
console.log('Failures detail:');
|
||||
for(const r of report){
|
||||
console.log(`- [${r.index}] ${r.name}:`);
|
||||
for(const e of r.errors) console.log(' -', e);
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(failures>0?1:0);
|
||||
}
|
||||
|
||||
run();
|
||||
Reference in New Issue
Block a user