refactor: Migrate database from SQLite to MariaDB
Database Migration: - Add MariaDB connection configuration and initialization (mariadb.js) - Create MariaDB schema update script (mariadb-schema-update.sql) - Add migration scripts for SQLite to MariaDB transition: * migrate-sqlite-to-mariadb.js: Main migration script * migrate-inventory-to-gear.js: Inventory schema migration - Remove SQLite-specific implementation files and databases Route Updates: - Update all route handlers to use MariaDB instead of SQLite - Migrate routes: playerRoutes, sessionRoutes, shopRoutes, bestiaryRoutes, rulesRoutes, rulesStagingRoutes, weaponsRoutes - Remove SQLite-specific route files (playerRoutes-sqlite.js, sessionRoutes-sqlite.js) - Update server.js to initialize MariaDB and register new weapon routes Backend Scripts: - Remove old SQLite migration scripts (migrate-to-sqlite.js, server-sqlite.js) - Delete obsolete database utility scripts from backup-scripts/ Frontend Updates: - Update logger utility for improved error handling and debugging - Enhance PlayerManagement component with better state management - Improve RequisitionShop component for MariaDB integration - Update DeathwatchRoller with performance improvements - Add login test suite (login.test.js) - Add XP progression utility (xpProgression.js) - Update dependencies in package.json and package-lock.json This migration improves: - Database scalability and performance - Transaction support for complex operations - Better data integrity and ACID compliance - Simplified deployment and backup procedures
This commit is contained in:
@@ -1,126 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
const https = require('https');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { db } = require('../database/sqlite-db');
|
||||
|
||||
function fetchUrl(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(url, { headers: { 'User-Agent': 'dwroller-bot/1.0' } }, (res) => {
|
||||
let data = '';
|
||||
res.on('data', (c) => data += c);
|
||||
res.on('end', () => resolve({ status: res.statusCode, body: data }));
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function sanitizeText(s) {
|
||||
if (!s) return '';
|
||||
let t = String(s);
|
||||
t = t.replace(/Explore More/ig, '');
|
||||
t = t.replace(/Skip to content/ig, '');
|
||||
t = t.replace(/40k-?RPG-?FFG Wiki/ig, '');
|
||||
t = t.replace(/Explore Main Page/ig, '');
|
||||
t = t.replace(/^(Category:|Special:|Local sitemap).*/gi, '');
|
||||
t = t.replace(/\[\d+\]/g, '');
|
||||
t = t.replace(/\s+/g, ' ').trim();
|
||||
t = t.replace(/^This (article|page) .*/i, '');
|
||||
return t.trim();
|
||||
}
|
||||
|
||||
function extractContentFromFandom(html) {
|
||||
const out = { paragraphs: [], sourceLines: [] };
|
||||
const m = html.match(/<div[^>]+class="mw-parser-output"[^>]*>([\s\S]*?)<div class="printfooter">/i);
|
||||
const block = m ? m[1] : html;
|
||||
|
||||
const pushText = (txt) => {
|
||||
if (!txt) return;
|
||||
let clean = txt.replace(/<[^>]+>/g, '').replace(/\[\d+\]/g, '').replace(/\s+/g, ' ').trim();
|
||||
if (!clean) return;
|
||||
if (/^Source[:\s]/i.test(clean)) { out.sourceLines.push(clean); return; }
|
||||
out.paragraphs.push(clean);
|
||||
};
|
||||
|
||||
const paraRe = /<p[^>]*>([\s\S]*?)<\/p>/ig;
|
||||
let p;
|
||||
while ((p = paraRe.exec(block)) !== null) pushText(p[1]);
|
||||
|
||||
if (out.paragraphs.length < 2) {
|
||||
const liRe = /<li[^>]*>([\s\S]*?)<\/li>/ig;
|
||||
while ((p = liRe.exec(block)) !== null) pushText(p[1]);
|
||||
}
|
||||
if (out.paragraphs.length < 2) {
|
||||
const ddRe = /<dd[^>]*>([\s\S]*?)<\/dd>/ig;
|
||||
while ((p = ddRe.exec(block)) !== null) pushText(p[1]);
|
||||
}
|
||||
if (out.paragraphs.length < 2) {
|
||||
const tdRe = /<td[^>]*>([\s\S]*?)<\/td>/ig;
|
||||
while ((p = tdRe.exec(block)) !== null) pushText(p[1]);
|
||||
}
|
||||
|
||||
out.paragraphs = out.paragraphs.filter(p => p && p.length > 20 && !/(?:Explore|Skip to content|Advertisement)/i.test(p));
|
||||
out.paragraphs = Array.from(new Set(out.paragraphs));
|
||||
return out;
|
||||
}
|
||||
|
||||
function findUseText(paragraphs) {
|
||||
for (const p of paragraphs) {
|
||||
if (/^(Use|Usage)[:\s]/i.test(p) || /\bUse[:\s]/i.test(p)) return p;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const mode = (process.argv[2] || 'preview').toLowerCase();
|
||||
if (!['preview','commit'].includes(mode)) { console.error('Mode must be preview or commit'); process.exit(1); }
|
||||
|
||||
const q = db.prepare("SELECT id, title, content FROM rules WHERE category = 'talents' AND (content IS NULL OR trim(content) = '' OR length(trim(content)) < 30) ORDER BY id");
|
||||
const rows = q.all();
|
||||
console.log('Found', rows.length, 'talents with missing/short content');
|
||||
const results = [];
|
||||
|
||||
for (const r of rows) {
|
||||
try {
|
||||
const title = (r.title || '').replace(/\s*\(Talent\)\s*$/i,'').trim();
|
||||
const urlTitle = encodeURIComponent(title.replace(/ /g, '_'));
|
||||
const url = `https://40k-rpg-ffg.fandom.com/wiki/${urlTitle}`;
|
||||
console.log('Fetching', title);
|
||||
const res = await fetchUrl(url);
|
||||
if (res.status !== 200) {
|
||||
console.warn('Failed to fetch', title, 'status', res.status);
|
||||
results.push({ id: r.id, title: r.title, status: 'fetch_failed', statusCode: res.status });
|
||||
continue;
|
||||
}
|
||||
const block = extractContentFromFandom(res.body);
|
||||
const orig = block.paragraphs || [];
|
||||
const paragraphs = orig.map(p => sanitizeText(p)).filter(Boolean);
|
||||
const main = paragraphs.length ? paragraphs[0] : '';
|
||||
const use = findUseText(paragraphs) || '';
|
||||
const descParts = paragraphs.filter(p => p !== main && p !== use);
|
||||
const description = descParts.join('\n\n');
|
||||
const newContent = [main, description, use ? `Use: ${use.replace(/^Use[:\s]*/i,'')}` : ''].filter(Boolean).join('\n\n');
|
||||
|
||||
results.push({ id: r.id, title: r.title, fetchedTitle: title, url, oldContent: r.content || '', newContent: newContent || '', extracted_paragraphs: orig });
|
||||
|
||||
if (mode === 'commit' && newContent && newContent.trim().length > 20) {
|
||||
const upd = db.prepare('UPDATE rules SET content = ?, source = ?, source_abbr = ? WHERE id = ?');
|
||||
upd.run(newContent, 'fandom', 'FAN', r.id);
|
||||
console.log('Updated id', r.id, title);
|
||||
}
|
||||
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
} catch (e) {
|
||||
console.error('Error processing id', r.id, e && e.message ? e.message : e);
|
||||
results.push({ id: r.id, title: r.title, status: 'error', error: e && e.message });
|
||||
}
|
||||
}
|
||||
|
||||
const ts = new Date().toISOString().replace(/[:.]/g,'-');
|
||||
const out = { generatedAt: new Date().toISOString(), mode, count: results.length, items: results };
|
||||
const outPath = path.join('/tmp', `talents_fill_${mode}_${ts}.json`);
|
||||
fs.writeFileSync(outPath, JSON.stringify(out, null, 2), 'utf8');
|
||||
console.log('Wrote report to', outPath);
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch(err => { console.error(err && err.stack ? err.stack : err); try{db.close()}catch(e){}; process.exit(1); });
|
||||
@@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const { stagingHelpers } = require('../database/sqlite-db');
|
||||
|
||||
const input = process.argv[2] || 'database/backups/sanitized-rules-test.json';
|
||||
if (!fs.existsSync(input)) {
|
||||
console.error('Input not found', input);
|
||||
process.exit(1);
|
||||
}
|
||||
const data = JSON.parse(fs.readFileSync(input,'utf8'));
|
||||
if (!data || !Array.isArray(data.sanitized)) {
|
||||
console.error('Expected file with { sanitized: [...] }');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
for (const item of data.sanitized) {
|
||||
try {
|
||||
stagingHelpers.insert(item);
|
||||
count++;
|
||||
} catch (e) {
|
||||
console.error('Failed insert staging', e && e.message);
|
||||
}
|
||||
}
|
||||
console.log('Imported to staging:', count);
|
||||
@@ -1,17 +0,0 @@
|
||||
const { db } = require('../database/sqlite-db');
|
||||
|
||||
function listSkills(limit = 50) {
|
||||
try {
|
||||
const rows = db.prepare(`SELECT rule_id as id, title, content, page, source, source_abbr as sourceAbbr, category FROM rules WHERE category = ? ORDER BY id LIMIT ?`).all('skills', limit);
|
||||
console.log(`Found ${rows.length} skill rows (showing up to ${limit}):`);
|
||||
rows.forEach((r, i) => {
|
||||
console.log(`${i + 1}. ${r.id} | ${r.title} | page=${r.page} | source=${r.source} | sourceAbbr=${r.sourceAbbr}`);
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Failed to query skills:', e);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
listSkills(200);
|
||||
@@ -1,166 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
|
||||
const dbPath = path.join(__dirname, '../database/sqlite/deathwatch.db');
|
||||
const backupPath = dbPath + '.backup.' + Date.now();
|
||||
|
||||
function safeReadJSON(p) {
|
||||
if (!fs.existsSync(p)) return null;
|
||||
try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch (e) { console.error('JSON parse error', p, e.message); return null; }
|
||||
}
|
||||
|
||||
console.log('=== MIGRATE JSON DATA INTO SQLITE ===');
|
||||
if (!fs.existsSync(dbPath)) {
|
||||
console.error('DB not found at', dbPath);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
fs.copyFileSync(dbPath, backupPath);
|
||||
console.log('Backup created:', backupPath);
|
||||
|
||||
const db = new Database(dbPath);
|
||||
|
||||
// Create consolidated tables
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS armour (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
req INTEGER DEFAULT 0,
|
||||
renown TEXT DEFAULT 'None',
|
||||
category TEXT,
|
||||
stats TEXT,
|
||||
source TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS weapons (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
req INTEGER DEFAULT 0,
|
||||
renown TEXT DEFAULT 'None',
|
||||
category TEXT,
|
||||
stats TEXT,
|
||||
source TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bestiary (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
book TEXT,
|
||||
page TEXT,
|
||||
pdf TEXT,
|
||||
stats TEXT,
|
||||
profile TEXT,
|
||||
snippet TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
rule_id TEXT UNIQUE,
|
||||
title TEXT,
|
||||
content TEXT,
|
||||
page INTEGER,
|
||||
source TEXT,
|
||||
source_abbr TEXT,
|
||||
category TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
const insertArmour = db.prepare(`INSERT OR IGNORE INTO armour (name, req, renown, category, stats, source) VALUES (?, ?, ?, ?, ?, ?)`);
|
||||
const insertWeapon = db.prepare(`INSERT OR IGNORE INTO weapons (name, req, renown, category, stats, source) VALUES (?, ?, ?, ?, ?, ?)`);
|
||||
const insertBestiary = db.prepare(`INSERT INTO bestiary (name, book, page, pdf, stats, profile, snippet) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
||||
const insertRule = db.prepare(`INSERT OR IGNORE INTO rules (rule_id, title, content, page, source, source_abbr, category) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
||||
|
||||
let totals = { armour:0, weapons:0, bestiary:0, rules:0 };
|
||||
|
||||
// Import armour
|
||||
const armourFile = path.join(__dirname, '../database/public/deathwatch-armor.json');
|
||||
const armourData = safeReadJSON(armourFile);
|
||||
if (armourData) {
|
||||
const categories = Object.keys(armourData);
|
||||
categories.forEach(cat => {
|
||||
const arr = armourData[cat];
|
||||
if (!Array.isArray(arr)) return;
|
||||
const insert = insertArmour;
|
||||
db.transaction(() => {
|
||||
for (const item of arr) {
|
||||
const stats = JSON.stringify(item.stats || {});
|
||||
const src = (item.stats && item.stats.source) || '';
|
||||
insert.run(item.name || '(unnamed)', item.req || 0, item.renown || 'None', item.category || cat, stats, src);
|
||||
totals.armour++;
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
// Import weapons
|
||||
const weaponsFile = path.join(__dirname, '../database/public/deathwatch-weapons-comprehensive.json');
|
||||
const weaponsData = safeReadJSON(weaponsFile);
|
||||
if (weaponsData) {
|
||||
// many weapon files use keys like rangedWeapons, meleeWeapons
|
||||
Object.keys(weaponsData).forEach(k => {
|
||||
const arr = weaponsData[k];
|
||||
if (!Array.isArray(arr)) return;
|
||||
db.transaction(() => {
|
||||
for (const w of arr) {
|
||||
const stats = JSON.stringify(w.stats || {});
|
||||
const src = (w.stats && w.stats.source) || '';
|
||||
insertWeapon.run(w.name || '(unnamed)', w.req || 0, w.renown || 'Any', w.category || k, stats, src);
|
||||
totals.weapons++;
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
// Import bestiary
|
||||
const bestiaryFile = path.join(__dirname, '../database/deathwatch-bestiary-extracted.json');
|
||||
const bestiaryData = safeReadJSON(bestiaryFile);
|
||||
if (bestiaryData && Array.isArray(bestiaryData.results)) {
|
||||
db.transaction(() => {
|
||||
for (const e of bestiaryData.results) {
|
||||
const stats = JSON.stringify(e.stats || {});
|
||||
const profile = JSON.stringify(e.profile || {});
|
||||
const name = e.bestiaryName || e.name || '(unnamed)';
|
||||
insertBestiary.run(name, e.book || '', e.page || '', e.pdf || '', stats, profile, e.stats && e.stats.snippet ? e.stats.snippet : '');
|
||||
totals.bestiary++;
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
// Import rules
|
||||
const rulesFile = path.join(__dirname, '../database/rules/rules-database.json');
|
||||
const rulesData = safeReadJSON(rulesFile);
|
||||
if (rulesData && Array.isArray(rulesData.rules)) {
|
||||
db.transaction(() => {
|
||||
for (const r of rulesData.rules) {
|
||||
insertRule.run(r.id || null, r.title || '', r.content || '', r.page || null, r.source || '', r.sourceAbbr || '', r.category || 'general');
|
||||
totals.rules++;
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
console.log('Import totals:', totals);
|
||||
|
||||
// Show row counts from DB for verification
|
||||
const counts = {
|
||||
armour: db.prepare('SELECT COUNT(*) as c FROM armour').get().c,
|
||||
weapons: db.prepare('SELECT COUNT(*) as c FROM weapons').get().c,
|
||||
bestiary: db.prepare('SELECT COUNT(*) as c FROM bestiary').get().c,
|
||||
rules: db.prepare('SELECT COUNT(*) as c FROM rules').get().c
|
||||
};
|
||||
|
||||
console.log('DB row counts:', counts);
|
||||
|
||||
// Print a small sample from each table
|
||||
console.log('\nSample armour:', db.prepare('SELECT name, category, stats FROM armour LIMIT 3').all());
|
||||
console.log('\nSample weapons:', db.prepare('SELECT name, category, stats FROM weapons LIMIT 3').all());
|
||||
console.log('\nSample bestiary:', db.prepare('SELECT name, book, snippet FROM bestiary LIMIT 3').all());
|
||||
console.log('\nSample rules:', db.prepare('SELECT rule_id, title FROM rules LIMIT 3').all());
|
||||
|
||||
db.close();
|
||||
console.log('\nMigration complete. DB backed up at', backupPath);
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const { db, logToFile } = require('../database/sqlite-db');
|
||||
|
||||
function nowTs() { return new Date().toISOString().replace(/[:.]/g,'-'); }
|
||||
|
||||
function titleCase(str) {
|
||||
return str.toLowerCase().split(/\s+/).map(w => {
|
||||
if (!w) return '';
|
||||
return w[0].toUpperCase() + w.slice(1);
|
||||
}).join(' ');
|
||||
}
|
||||
|
||||
function cleanTitle(title) {
|
||||
if (!title) return title;
|
||||
const letters = title.replace(/[^A-Za-z]/g,'');
|
||||
const uppers = (title.match(/[A-Z]/g) || []).length;
|
||||
// if mostly uppercase, convert to title case
|
||||
if (letters && (uppers / letters.length) > 0.5) {
|
||||
return titleCase(title.replace(/\s+/g,' ').trim());
|
||||
}
|
||||
// otherwise trim
|
||||
return title.trim();
|
||||
}
|
||||
|
||||
function cleanContent(text) {
|
||||
if (!text) return text;
|
||||
let s = String(text);
|
||||
s = s.replace(/\r\n/g, '\n');
|
||||
s = s.replace(/[ \t]+/g, ' ');
|
||||
// remove hyphenation at line breaks
|
||||
s = s.replace(/-\n\s*/g, '');
|
||||
// collapse more than 2 newlines into paragraph breaks
|
||||
s = s.replace(/\n{3,}/g, '\n\n');
|
||||
// join lines that look like soft-wrapped lines: a line break between
|
||||
// a non-punctuation end and a lowercase/digit start
|
||||
s = s.replace(/([^\.\!\?\:\;\"\'\)\]\}])\n(\s*[a-z0-9])/g, '$1 $2');
|
||||
// trim spaces at start/end of lines
|
||||
s = s.split('\n').map(l => l.trim()).join('\n');
|
||||
// collapse repeated spaces
|
||||
s = s.replace(/ {2,}/g, ' ');
|
||||
// trim overall
|
||||
s = s.trim();
|
||||
return s;
|
||||
}
|
||||
|
||||
function backupRules(rows) {
|
||||
const backupDir = path.join(__dirname, '..', 'database', 'backups');
|
||||
if (!fs.existsSync(backupDir)) fs.mkdirSync(backupDir, { recursive: true });
|
||||
const file = path.join(backupDir, `rules-backup-${nowTs()}.json`);
|
||||
fs.writeFileSync(file, JSON.stringify({ backedAt: new Date().toISOString(), count: rows.length, rows }, null, 2), 'utf8');
|
||||
return file;
|
||||
}
|
||||
|
||||
function main() {
|
||||
console.log('Backing up rules table and normalizing content...');
|
||||
const rows = db.prepare('SELECT id, title, content FROM rules').all();
|
||||
if (!rows || rows.length === 0) {
|
||||
console.log('No rules found in DB. Exiting.');
|
||||
return;
|
||||
}
|
||||
const backupFile = backupRules(rows);
|
||||
console.log('Backup written to', backupFile);
|
||||
|
||||
const updateStmt = db.prepare('UPDATE rules SET title = ?, content = ? WHERE id = ?');
|
||||
let changed = 0;
|
||||
db.transaction(() => {
|
||||
for (const r of rows) {
|
||||
const cleanedTitle = cleanTitle(r.title || '');
|
||||
const cleanedContent = cleanContent(r.content || '');
|
||||
if ((cleanedTitle !== (r.title||'').trim()) || (cleanedContent !== (r.content||'').trim())) {
|
||||
updateStmt.run(cleanedTitle, cleanedContent, r.id);
|
||||
changed++;
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
console.log(`Normalization complete. Rows updated: ${changed}`);
|
||||
logToFile('normalize-rules-db: completed', { updated: changed });
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -1,48 +0,0 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { db } = require('../database/sqlite-db');
|
||||
|
||||
const outDir = path.join(__dirname, '..', 'database', 'backups');
|
||||
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
|
||||
try {
|
||||
// Select rows to purge: any rule where source != 'csv-import' OR page contains 'p.1' (case-insensitive)
|
||||
const rowsToPurge = db.prepare("SELECT * FROM rules WHERE source != ? OR (page IS NOT NULL AND lower(page) LIKE '%p.1%')").all('csv-import');
|
||||
console.log('Found rows to purge:', rowsToPurge.length);
|
||||
const backupFile = path.join(outDir, `purge_non_csv_rules_backup_${ts}.json`);
|
||||
fs.writeFileSync(backupFile, JSON.stringify({ purgedAt: new Date().toISOString(), count: rowsToPurge.length, rows: rowsToPurge }, null, 2), 'utf8');
|
||||
console.log('Backup written to', backupFile);
|
||||
|
||||
if (rowsToPurge.length === 0) {
|
||||
console.log('Nothing to purge');
|
||||
db.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Delete by id in transaction
|
||||
const del = db.prepare('DELETE FROM rules WHERE id = ?');
|
||||
db.transaction(() => {
|
||||
for (const r of rowsToPurge) {
|
||||
del.run(r.id);
|
||||
}
|
||||
})();
|
||||
|
||||
const remaining = db.prepare("SELECT source, COUNT(*) as c FROM rules GROUP BY source ORDER BY c DESC").all();
|
||||
console.log('Remaining rows by source:', remaining);
|
||||
|
||||
const total = db.prepare('SELECT COUNT(*) as c FROM rules').get().c;
|
||||
console.log('Total rules now in DB:', total);
|
||||
|
||||
const report = { purgedAt: new Date().toISOString(), purgedCount: rowsToPurge.length, remaining, total };
|
||||
fs.writeFileSync(path.join(outDir, `purge_non_csv_rules_report_${ts}.json`), JSON.stringify(report, null, 2), 'utf8');
|
||||
console.log('Purge report written');
|
||||
|
||||
} catch (e) {
|
||||
console.error('Error during purge:', e);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
console.log('Done');
|
||||
@@ -1,159 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
const https = require('https');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { db } = require('../database/sqlite-db');
|
||||
|
||||
function fetchUrl(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
https.get(url, { headers: { 'User-Agent': 'dwroller-bot/1.0' } }, (res) => {
|
||||
let data = '';
|
||||
res.on('data', (c) => data += c);
|
||||
res.on('end', () => resolve({ status: res.statusCode, body: data }));
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function extractContentFromFandom(html) {
|
||||
const out = { paragraphs: [], headings: [], sourceLines: [] };
|
||||
const m = html.match(/<div[^>]+class="mw-parser-output"[^>]*>([\s\S]*?)<div class="printfooter">/i);
|
||||
const block = m ? m[1] : html;
|
||||
|
||||
const pushText = (txt) => {
|
||||
if (!txt) return;
|
||||
let clean = txt.replace(/<[^>]+>/g, '')
|
||||
.replace(/\[\d+\]/g, '')
|
||||
.replace(/\s+/g, ' ').trim();
|
||||
if (!clean) return;
|
||||
if (/^Source[:\s]/i.test(clean)) {
|
||||
out.sourceLines.push(clean);
|
||||
return;
|
||||
}
|
||||
out.paragraphs.push(clean);
|
||||
};
|
||||
|
||||
const paraRe = /<p[^>]*>([\s\S]*?)<\/p>/ig;
|
||||
let p;
|
||||
while ((p = paraRe.exec(block)) !== null) pushText(p[1]);
|
||||
|
||||
if (out.paragraphs.length < 2) {
|
||||
const liRe = /<li[^>]*>([\s\S]*?)<\/li>/ig;
|
||||
while ((p = liRe.exec(block)) !== null) pushText(p[1]);
|
||||
}
|
||||
if (out.paragraphs.length < 2) {
|
||||
const ddRe = /<dd[^>]*>([\s\S]*?)<\/dd>/ig;
|
||||
while ((p = ddRe.exec(block)) !== null) pushText(p[1]);
|
||||
}
|
||||
if (out.paragraphs.length < 2) {
|
||||
const tdRe = /<td[^>]*>([\s\S]*?)<\/td>/ig;
|
||||
while ((p = tdRe.exec(block)) !== null) pushText(p[1]);
|
||||
}
|
||||
|
||||
const hRe = /<h[2-3][^>]*>([\s\S]*?)<\/h[2-3]>/ig;
|
||||
let h;
|
||||
while ((h = hRe.exec(block)) !== null) {
|
||||
const ht = h[1].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
|
||||
if (ht.length) out.headings.push(ht);
|
||||
}
|
||||
|
||||
const noiseRe = /(?:Explore|Skip to content|Advertisement|History|Main Page|Discuss|Community|Interactive Maps|Recently Changed|Explore More|All Pages|Pages|Recent Blog Posts|Recently Changed Pages|Explore Main Page)/i;
|
||||
out.paragraphs = out.paragraphs.filter(p => {
|
||||
if (noiseRe.test(p)) return false;
|
||||
if (p.length < 30) return false;
|
||||
if (/^\s*\w+(\s+\w+){0,2}\s*$/.test(p) && p.split(' ').length <= 3) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
out.paragraphs = Array.from(new Set(out.paragraphs));
|
||||
return out;
|
||||
}
|
||||
|
||||
function sanitizeText(s) {
|
||||
if (!s) return '';
|
||||
let t = String(s);
|
||||
t = t.replace(/Explore More/ig, '');
|
||||
t = t.replace(/Skip to content/ig, '');
|
||||
t = t.replace(/40k-?RPG-?FFG Wiki/ig, '');
|
||||
t = t.replace(/Explore Main Page/ig, '');
|
||||
t = t.replace(/^(Category:|Special:|Local sitemap).*/gi, '');
|
||||
t = t.replace(/\[\d+\]/g, '');
|
||||
t = t.replace(/\s+/g, ' ').trim();
|
||||
t = t.replace(/^This (article|page) .*/i, '');
|
||||
return t.trim();
|
||||
}
|
||||
|
||||
async function repair({ commit = false } = {}) {
|
||||
// find problem rows: contain common nav noise OR very short content OR content starts with 'Source:' only
|
||||
const q = `SELECT title, content FROM rules WHERE category='skills' AND (content LIKE '%Explore More%' OR content LIKE '%Skip to content%' OR length(content) < 120 OR content LIKE 'Source:%' ) ORDER BY title`;
|
||||
const rows = db.prepare(q).all();
|
||||
console.log('Found', rows.length, 'skills to inspect');
|
||||
if (!rows.length) { db.close(); return; }
|
||||
|
||||
const results = [];
|
||||
for (const r of rows) {
|
||||
try {
|
||||
const title = r.title;
|
||||
const urlTitle = encodeURIComponent(title.replace(/ /g, '_'));
|
||||
const url = `https://40k-rpg-ffg.fandom.com/wiki/${urlTitle}`;
|
||||
console.log('Fetching', title);
|
||||
const res = await fetchUrl(url);
|
||||
if (res.status !== 200) {
|
||||
console.warn('Fetch failed', title, res.status);
|
||||
continue;
|
||||
}
|
||||
const block = extractContentFromFandom(res.body);
|
||||
const paras = block.paragraphs.map(sanitizeText).filter(Boolean);
|
||||
const use = paras.find(p => /^(Use|Usage)[:\s]/i.test(p)) || '';
|
||||
const descParts = paras.filter(p => p !== use);
|
||||
let newContent = '';
|
||||
if (descParts.length) {
|
||||
// prefer the longest paragraph as primary
|
||||
const primary = descParts.reduce((a,b)=> a.length>=b.length?a:b,'');
|
||||
const others = descParts.filter(p=>p!==primary);
|
||||
newContent = [primary, others.join('\n\n')].filter(Boolean).join('\n\n');
|
||||
}
|
||||
if (block.sourceLines && block.sourceLines.length) {
|
||||
newContent = (newContent ? newContent + '\n\n' : '') + block.sourceLines.join(' | ');
|
||||
}
|
||||
if (!newContent) {
|
||||
// nothing useful extracted, skip
|
||||
console.log('No useful content for', title);
|
||||
continue;
|
||||
}
|
||||
results.push({ title, old: r.content, newContent, url });
|
||||
} catch (e) {
|
||||
console.error('Error for', r.title, e && e.message);
|
||||
}
|
||||
}
|
||||
|
||||
const ts = new Date().toISOString().replace(/[:.]/g,'-');
|
||||
const previewPath = path.join('/tmp', `repair_skills_preview_${ts}.json`);
|
||||
fs.writeFileSync(previewPath, JSON.stringify({ generatedAt: new Date().toISOString(), count: results.length, items: results }, null, 2), 'utf8');
|
||||
console.log('Wrote preview to', previewPath);
|
||||
|
||||
if (!commit) { db.close(); return; }
|
||||
|
||||
// apply updates
|
||||
const update = db.prepare('UPDATE rules SET content = ?, source = ?, source_abbr = ? WHERE title = ?');
|
||||
let changed = 0;
|
||||
db.transaction(() => {
|
||||
for (const it of results) {
|
||||
try {
|
||||
update.run(it.newContent, 'https://40k-rpg-ffg.fandom.com', 'fandom', it.title);
|
||||
changed++;
|
||||
} catch (e) {
|
||||
console.error('Failed update', it.title, e && e.message);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
const commitPath = path.join('/tmp', `repair_skills_committed_${ts}.json`);
|
||||
fs.writeFileSync(commitPath, JSON.stringify({ committedAt: new Date().toISOString(), changed, items: results.map(r=>({title:r.title})) }, null, 2), 'utf8');
|
||||
console.log('Committed', changed, 'rows. Details in', commitPath);
|
||||
db.close();
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
const commit = (process.argv[2] === '--commit');
|
||||
repair({ commit }).then(()=>process.exit(0)).catch(err=>{ console.error(err); db.close(); process.exit(1); });
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { db } = require('../database/sqlite-db');
|
||||
|
||||
function nowTs() { return new Date().toISOString().replace(/[:.]/g,'-'); }
|
||||
|
||||
function charEntropy(s) {
|
||||
if (!s || s.length === 0) return 0;
|
||||
const freq = {};
|
||||
for (const ch of s) freq[ch] = (freq[ch]||0) + 1;
|
||||
const len = s.length;
|
||||
let ent = 0;
|
||||
for (const k in freq) {
|
||||
const p = freq[k]/len;
|
||||
ent -= p * Math.log2(p);
|
||||
}
|
||||
return ent;
|
||||
}
|
||||
|
||||
function scoreText(text) {
|
||||
if (!text) return {score:0,metrics:{}};
|
||||
const s = String(text);
|
||||
const length = s.length;
|
||||
const letters = s.replace(/[^A-Za-z]/g,'');
|
||||
const upper = (s.match(/[A-Z]/g)||[]).length;
|
||||
const digits = (s.match(/[0-9]/g)||[]).length;
|
||||
const nonAlphaNum = (s.match(/[^A-Za-z0-9\s\.,;:\'"\-()\[\]\/\\]/g)||[]).length;
|
||||
const punctuation = (s.match(/[\.,;:\!\?\-\(\)\[\]"\']/g)||[]).length;
|
||||
const newlines = (s.match(/\n/g)||[]).length;
|
||||
const lines = s.split(/\n/);
|
||||
const shortLines = lines.filter(l => l.trim().length > 0 && l.trim().length < 40).length;
|
||||
const avgWordLen = (s.match(/\w+/g)||[]).reduce((a,w)=>a+w.length,0)/Math.max(1,(s.match(/\w+/g)||[]).length);
|
||||
const entropy = charEntropy(s);
|
||||
|
||||
const upperRatio = letters.length ? upper/letters.length : 0;
|
||||
const nonAlphaRatio = length ? nonAlphaNum/length : 0;
|
||||
const newlineDensity = length ? newlines/length : 0;
|
||||
const shortLineRatio = lines.length ? shortLines/lines.length : 0;
|
||||
const punctDensity = length ? punctuation/length : 0;
|
||||
|
||||
// Score: higher for uppercase-heavy, non-alpha junk, many newlines, many short lines, low avg word length, high entropy
|
||||
// Weights chosen empirically to bring noisy texts to the top.
|
||||
const score = (
|
||||
upperRatio * 2.5 +
|
||||
nonAlphaRatio * 4.0 +
|
||||
newlineDensity * 3.0 +
|
||||
shortLineRatio * 1.6 +
|
||||
(1/Math.max(1, avgWordLen)) * 1.2 +
|
||||
(entropy/6.0) * 1.0 +
|
||||
punctDensity * 0.8
|
||||
) * 100;
|
||||
|
||||
return { score, metrics: { length, upper, letters: letters.length, upperRatio, nonAlphaNum, nonAlphaRatio, newlines, newlineDensity, lines: lines.length, shortLines, shortLineRatio, avgWordLen, entropy, punctDensity } };
|
||||
}
|
||||
|
||||
function main() {
|
||||
const rows = db.prepare('SELECT id, title, page, source, content FROM rules').all();
|
||||
if (!rows || rows.length === 0) {
|
||||
console.log('No rules found');
|
||||
return;
|
||||
}
|
||||
const scored = rows.map(r => {
|
||||
const text = (r.title||'') + '\n' + (r.content||'');
|
||||
const res = scoreText(text);
|
||||
return { id: r.id, title: (r.title||'').trim(), page: r.page, source: r.source, score: Math.round(res.score*100)/100, metrics: res.metrics, snippet: (r.content||'').replace(/\n/g,' ').slice(0,240) };
|
||||
});
|
||||
scored.sort((a,b)=>b.score - a.score);
|
||||
|
||||
const top = scored.slice(0,40);
|
||||
const outDir = path.join(__dirname,'..','database','backups');
|
||||
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
||||
const outFile = path.join(outDir, `rules-noise-report-${nowTs()}.json`);
|
||||
fs.writeFileSync(outFile, JSON.stringify({ generatedAt: new Date().toISOString(), count: scored.length, top }, null, 2), 'utf8');
|
||||
|
||||
console.log('Noise scoring complete. Total rules:', scored.length);
|
||||
console.log('Report written to', outFile);
|
||||
console.log('\nTop 25 noisy rules:');
|
||||
top.slice(0,25).forEach((r,i)=>{
|
||||
console.log(`${String(i+1).padStart(2,' ')}. id=${r.id} score=${r.score} title="${r.title}" page=${r.page} source=${r.source}`);
|
||||
console.log(' snippet:', r.snippet.replace(/\s+/g,' ').slice(0,200));
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -1,150 +0,0 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { db } = require('../database/sqlite-db');
|
||||
|
||||
if (process.argv.length < 3) {
|
||||
console.error('Usage: node scripts/sync-skills-csv-to-db.js <csv-path>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const csvPath = process.argv[2];
|
||||
if (!fs.existsSync(csvPath)) {
|
||||
console.error('CSV file not found:', csvPath);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function parseCSV(content) {
|
||||
// Minimal RFC4180-ish parser supporting quoted fields and commas
|
||||
const lines = [];
|
||||
let cur = '';
|
||||
let inQuotes = false;
|
||||
for (let i = 0; i < content.length; i++) {
|
||||
const ch = content[i];
|
||||
const nxt = content[i + 1];
|
||||
if (ch === '"') {
|
||||
if (inQuotes && nxt === '"') { // escaped quote
|
||||
cur += '"';
|
||||
i++; // skip next
|
||||
} else {
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (ch === '\n' && !inQuotes) {
|
||||
lines.push(cur);
|
||||
cur = '';
|
||||
continue;
|
||||
}
|
||||
cur += ch;
|
||||
}
|
||||
if (cur.length) lines.push(cur);
|
||||
|
||||
return lines.map(l => {
|
||||
const cols = [];
|
||||
let cell = '';
|
||||
let q = false;
|
||||
for (let i = 0; i < l.length; i++) {
|
||||
const ch = l[i];
|
||||
const nx = l[i + 1];
|
||||
if (ch === '"') {
|
||||
if (q && nx === '"') { cell += '"'; i++; continue; }
|
||||
q = !q; continue;
|
||||
}
|
||||
if (ch === ',' && !q) { cols.push(cell); cell = ''; continue; }
|
||||
cell += ch;
|
||||
}
|
||||
cols.push(cell);
|
||||
return cols.map(c => c.trim());
|
||||
});
|
||||
}
|
||||
|
||||
function slugify(s) {
|
||||
return String(s || '')
|
||||
.toLowerCase()
|
||||
.normalize('NFKD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
}
|
||||
|
||||
const outDir = path.join(__dirname, '..', 'database', 'backups');
|
||||
if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
|
||||
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
|
||||
try {
|
||||
const raw = fs.readFileSync(csvPath, 'utf8');
|
||||
const rows = parseCSV(raw);
|
||||
if (rows.length < 2) {
|
||||
console.error('No CSV rows found');
|
||||
process.exit(1);
|
||||
}
|
||||
const headers = rows[0].map(h => h.toLowerCase());
|
||||
const data = rows.slice(1).map(r => {
|
||||
const obj = {};
|
||||
for (let i = 0; i < headers.length; i++) obj[headers[i]] = r[i] || '';
|
||||
return obj;
|
||||
}).filter(d => (d.name || '').trim());
|
||||
|
||||
// Backup existing skills rows
|
||||
const backupFile = path.join(outDir, `rules_skills_backup_${ts}.json`);
|
||||
const existing = db.prepare('SELECT * FROM rules WHERE category = ?').all('skills');
|
||||
fs.writeFileSync(backupFile, JSON.stringify({ backedAt: new Date().toISOString(), count: existing.length, rows: existing }, null, 2), 'utf8');
|
||||
console.log('Backup written to', backupFile, ' (rows:', existing.length, ')');
|
||||
|
||||
// Delete existing skill rows
|
||||
const del = db.prepare('DELETE FROM rules WHERE category = ?');
|
||||
const delRes = del.run('skills');
|
||||
console.log('Deleted rules where category=skills, changes:', delRes.changes);
|
||||
|
||||
// Prepare statements: delete any conflicting rule_id and insert
|
||||
const deleteById = db.prepare('DELETE FROM rules WHERE rule_id = ?');
|
||||
const insert = db.prepare('INSERT INTO rules (rule_id, title, content, page, source, source_abbr, category) VALUES (?, ?, ?, ?, ?, ?, ?)');
|
||||
|
||||
const inserted = [];
|
||||
db.transaction(() => {
|
||||
for (const r of data) {
|
||||
const name = (r.name || '').trim();
|
||||
const skill_text = (r.skill_text || '').trim();
|
||||
const skill_description = (r.skill_description || '').trim();
|
||||
const skill_use = (r.skill_use || '').trim();
|
||||
const content = [skill_text, skill_description, skill_use ? `Use: ${skill_use}` : ''].filter(Boolean).join('\n\n');
|
||||
const rule_id = slugify(name);
|
||||
// Remove any existing row with this rule_id (ensures 1:1 mapping to CSV)
|
||||
try {
|
||||
deleteById.run(rule_id);
|
||||
} catch (e) {
|
||||
// ignore
|
||||
}
|
||||
insert.run(rule_id, name, content || '', null, 'csv-import', 'CSV', 'skills');
|
||||
inserted.push(rule_id);
|
||||
}
|
||||
})();
|
||||
|
||||
const newCount = db.prepare('SELECT COUNT(*) as c FROM rules WHERE category = ?').get('skills').c;
|
||||
console.log('Inserted rows from CSV:', inserted.length, 'DB now has skills rows:', newCount);
|
||||
if (newCount !== inserted.length) {
|
||||
console.warn('Count mismatch: inserted', inserted.length, 'but DB count is', newCount);
|
||||
}
|
||||
|
||||
// Ensure only CSV-sourced skills exist (sanity check)
|
||||
const nonCsv = db.prepare("SELECT COUNT(*) as c FROM rules WHERE category = ? AND source != ?").get('skills', 'csv-import').c;
|
||||
console.log('Non-CSV skill rows remaining:', nonCsv);
|
||||
|
||||
// Output sample first 10
|
||||
const sample = db.prepare('SELECT rule_id, title FROM rules WHERE category = ? ORDER BY id LIMIT 10').all('skills');
|
||||
console.log('Sample rows:');
|
||||
sample.forEach((s, i) => console.log(`${i + 1}. ${s.rule_id} | ${s.title}`));
|
||||
|
||||
// final verification: write sync report
|
||||
const report = { syncedAt: new Date().toISOString(), csvRows: data.length, inserted: inserted.length, dbSkills: newCount };
|
||||
fs.writeFileSync(path.join(outDir, `rules_skills_sync_report_${ts}.json`), JSON.stringify(report, null, 2), 'utf8');
|
||||
console.log('Sync report written');
|
||||
|
||||
} catch (e) {
|
||||
console.error('Failure during sync:', e);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
console.log('Done');
|
||||
Reference in New Issue
Block a user