224 lines
8.3 KiB
JavaScript
224 lines
8.3 KiB
JavaScript
const fs = require('fs');
|
||
const path = require('path');
|
||
const { pool } = require('../mariadb');
|
||
|
||
const APPLY = process.argv.includes('--apply');
|
||
const MAX_LINES = 8;
|
||
const MAX_CHARS = 900;
|
||
|
||
const ROOT = path.join(__dirname, '..');
|
||
const BACKUP_DIR = path.join(ROOT, 'backups');
|
||
const JSON_RULES = path.join(ROOT, 'rules', 'rules-database.json');
|
||
|
||
const BAD_TITLE_RE = /^([_\-—–.·•\s\d()+%†]+|NAME|SKILLS|CHARACTERISTICS|SQUAD MODE|COHESION:|SQUAD DESIGNATION:|ISBN:.*)$/i;
|
||
const STAT_TITLE_RE = /^(\(?\d+[-–]?\d*\)?(\s+\(?[\d–-]+\)?){1,}|.*Pen\s+\d+.*|.*Reload.*|.*Tearing.*|.*Unnatural .*x2.*)$/i;
|
||
const NOISE_LINE_RE = /^(\d{1,3}|chapter\s+[ivx]+|[ivx]+:\s+|table\s+\d|page\s+\d|www\.|isbn|™|®|©|credits|contents|printed in|fantasy flight|games workshop|character name:|player name:|effects:|type:\s*✔|trained\s*$|basic\s*$|advanced\s+skills|actioN descriptioNs)$/i;
|
||
const SHEET_RE = /(SPACE MARINE ABILITIES|POWER ARMOUR ABILITIES|PrimarCh’s Curse|Battle Fatigue|CurreNt PoiNts|weapon skill\s+ballistic skill|character name:\s+player name)/i;
|
||
const STATBLOCK_RE = /(Movement:\s*\d|Wounds:\s*\d|Skills:.*Talents:|Armour:.*Weapons:|wS\s+BS\s+S\s+T\s+ag\s+int\s+Per\s+WP\s+fel)/i;
|
||
const TABLE_NOISE_RE = /(Rank \d .* Advances|Advance\s+Cost\s+Type|Objective\/Difficulty\s+Requisition|Name\s+Wt\s+Req\s+Renown)/i;
|
||
const MECHANICS_RE = /(Test|Action|Reaction|Damage|Wounds?|Skill|Talent|Trait|bonus|penalty|Requisition|Renown|Cohesion|Fear|Insanity|Horde|Armou?r|Weapon|Psychic|Initiative|Critical|Fatigue|Pinning|Prone|Dodge|Parry|Opposed|Difficulty|Full Action|Half Action|Free Action|Toughness|Willpower|Agility|Fellowship|Ballistic|Weapon Skill)/i;
|
||
const LORE_TITLE_RE = /^(GAME|THE LION|SANGUINIUS|LEMAN RUSS|THE DEATHWING|THE RAVENWING|THE LION AND THE WOLF|IMPERIUM|ULTIMA|SEGEMENTUM|THE AGES OF HUMANITY|THE SOUL BINDING|EXTERMINATUS|RADICALS AND PURITANS|BLACK SHIELDS|CATECHISM OF THE XENOS|THE HADEX ANOMALY|OF THE 41ST MILLENNIUM|WE ARE THE IMPERIUM)/i;
|
||
|
||
function cleanWhitespace(text) {
|
||
return String(text || '')
|
||
.replace(/\r\n/g, '\n')
|
||
.replace(/[\x00-\x09\x0B\x0C\x0E-\x1F]/g, '')
|
||
.replace(/[ \t]+/g, ' ')
|
||
.replace(/\n\s+/g, '\n')
|
||
.replace(/\s+\n/g, '\n')
|
||
.replace(/\n{3,}/g, '\n\n')
|
||
.trim();
|
||
}
|
||
|
||
function cleanTitle(title) {
|
||
return cleanWhitespace(title)
|
||
.replace(/[〔〕]/g, '')
|
||
.replace(/\s+/g, ' ')
|
||
.trim();
|
||
}
|
||
|
||
function isBadTitle(title) {
|
||
const t = cleanTitle(title);
|
||
if (!t || t.length < 3) return true;
|
||
if (BAD_TITLE_RE.test(t)) return true;
|
||
if (STAT_TITLE_RE.test(t)) return true;
|
||
if (/^(Prerequisites:|[–-][A-Z]\.|†+|[0-9]{2,}|[0-9]+[–-][0-9]+)$/i.test(t)) return true;
|
||
return false;
|
||
}
|
||
|
||
function noiseScore(rule) {
|
||
const title = cleanTitle(rule.title);
|
||
const content = cleanWhitespace(rule.content);
|
||
let score = 0;
|
||
if (isBadTitle(title)) score += 4;
|
||
if (SHEET_RE.test(content) || SHEET_RE.test(title)) score += 8;
|
||
if (STATBLOCK_RE.test(content) || STATBLOCK_RE.test(title)) score += 5;
|
||
if (TABLE_NOISE_RE.test(content) || TABLE_NOISE_RE.test(title)) score += 4;
|
||
if (content.length > 1300) score += 2;
|
||
const lines = content.split('\n').filter(Boolean);
|
||
const noisyLines = lines.filter(line => NOISE_LINE_RE.test(line) || /^\W+$/.test(line) || line.length > 160).length;
|
||
if (lines.length && noisyLines / lines.length > 0.45) score += 4;
|
||
if ((title.match(/\b[A-Z]{2,}\b/g) || []).length > 3) score += 2;
|
||
return score;
|
||
}
|
||
|
||
function meaningfulLines(rule) {
|
||
const title = cleanTitle(rule.title);
|
||
const content = cleanWhitespace(rule.content);
|
||
const raw = content.split('\n').map(line => cleanWhitespace(line)).filter(Boolean);
|
||
const lines = [];
|
||
for (const line of raw) {
|
||
if (!line || line === title) continue;
|
||
if (NOISE_LINE_RE.test(line)) continue;
|
||
if (/^\W+$/.test(line)) continue;
|
||
if (/^[A-Z][A-Za-z\s]+:\s*$/.test(line) && line.length < 24) continue;
|
||
if (line.length > 180 && !/[.;:]$/.test(line)) continue;
|
||
if (/(\s{8,}| {4,})/.test(line)) continue;
|
||
if (/^(S|T|ag|int|per|wp|fel|wS|BS)(\s+\S+){4,}$/i.test(line)) continue;
|
||
lines.push(line);
|
||
}
|
||
return lines;
|
||
}
|
||
|
||
function sanitizeContent(rule) {
|
||
const summary = cleanWhitespace(rule.summary);
|
||
const lines = meaningfulLines(rule);
|
||
const picked = [];
|
||
|
||
if (summary && summary.length > 20 && !SHEET_RE.test(summary)) {
|
||
picked.push(summary);
|
||
}
|
||
|
||
for (const line of lines) {
|
||
if (picked.join('\n').length > MAX_CHARS) break;
|
||
if (picked.some(existing => existing.toLowerCase() === line.toLowerCase())) continue;
|
||
picked.push(line);
|
||
if (picked.length >= MAX_LINES) break;
|
||
}
|
||
|
||
let content = picked.join('\n').trim();
|
||
if (content.length > MAX_CHARS) {
|
||
content = content.slice(0, MAX_CHARS).replace(/\s+\S*$/, '').trim();
|
||
}
|
||
return content;
|
||
}
|
||
|
||
function shouldDelete(rule) {
|
||
const title = cleanTitle(rule.title);
|
||
const content = cleanWhitespace(rule.content);
|
||
const score = noiseScore(rule);
|
||
const useful = sanitizeContent(rule);
|
||
if (LORE_TITLE_RE.test(title) && !MECHANICS_RE.test(useful)) return true;
|
||
if (LORE_TITLE_RE.test(title) && String(rule.category || '').toLowerCase() === 'general') return true;
|
||
if (!useful || useful.length < 35) return true;
|
||
if (SHEET_RE.test(content)) return true;
|
||
if (score >= 10) return true;
|
||
if (isBadTitle(title) && score >= 7) return true;
|
||
return false;
|
||
}
|
||
|
||
function sanitizeRule(rule) {
|
||
return {
|
||
...rule,
|
||
title: cleanTitle(rule.title),
|
||
content: sanitizeContent(rule),
|
||
summary: cleanWhitespace(rule.summary),
|
||
};
|
||
}
|
||
|
||
function backupFileName(prefix) {
|
||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||
return path.join(BACKUP_DIR, `${prefix}-${stamp}.json`);
|
||
}
|
||
|
||
async function main() {
|
||
fs.mkdirSync(BACKUP_DIR, { recursive: true });
|
||
|
||
const [rows] = await pool.execute('SELECT * FROM rules ORDER BY id');
|
||
const dbBackupPath = backupFileName('rules-table-before-sanitize');
|
||
fs.writeFileSync(dbBackupPath, JSON.stringify(rows, null, 2));
|
||
|
||
const deletes = [];
|
||
const updates = [];
|
||
for (const row of rows) {
|
||
if (shouldDelete(row)) {
|
||
deletes.push(row);
|
||
continue;
|
||
}
|
||
const next = sanitizeRule(row);
|
||
if (
|
||
next.title !== row.title ||
|
||
next.content !== row.content ||
|
||
next.summary !== row.summary
|
||
) {
|
||
updates.push(next);
|
||
}
|
||
}
|
||
|
||
let jsonStats = null;
|
||
if (fs.existsSync(JSON_RULES)) {
|
||
const json = JSON.parse(fs.readFileSync(JSON_RULES, 'utf8'));
|
||
const rules = Array.isArray(json.rules) ? json.rules : [];
|
||
const jsonBackupPath = backupFileName('rules-json-before-sanitize');
|
||
fs.writeFileSync(jsonBackupPath, JSON.stringify(json, null, 2));
|
||
const kept = [];
|
||
let jsonDeleted = 0;
|
||
let jsonUpdated = 0;
|
||
for (const rule of rules) {
|
||
if (shouldDelete(rule)) {
|
||
jsonDeleted += 1;
|
||
continue;
|
||
}
|
||
const next = sanitizeRule(rule);
|
||
if (next.content !== rule.content || next.title !== rule.title || next.summary !== rule.summary) jsonUpdated += 1;
|
||
kept.push(next);
|
||
}
|
||
jsonStats = { before: rules.length, after: kept.length, deleted: jsonDeleted, updated: jsonUpdated, backup: jsonBackupPath };
|
||
if (APPLY) fs.writeFileSync(JSON_RULES, JSON.stringify({ ...json, rules: kept }, null, 2));
|
||
}
|
||
|
||
console.log(JSON.stringify({
|
||
apply: APPLY,
|
||
db: {
|
||
before: rows.length,
|
||
after: rows.length - deletes.length,
|
||
deleted: deletes.length,
|
||
updated: updates.length,
|
||
backup: dbBackupPath,
|
||
deleteSamples: deletes.slice(0, 12).map(r => ({ id: r.id, title: r.title, score: noiseScore(r) })),
|
||
updateSamples: updates.slice(0, 8).map(r => ({ id: r.id, title: r.title, chars: r.content.length, preview: r.content.slice(0, 160) })),
|
||
},
|
||
json: jsonStats,
|
||
}, null, 2));
|
||
|
||
if (APPLY) {
|
||
const conn = await pool.getConnection();
|
||
try {
|
||
await conn.beginTransaction();
|
||
for (const row of updates) {
|
||
await conn.execute(
|
||
'UPDATE rules SET title = ?, content = ?, summary = ? WHERE id = ?',
|
||
[row.title, row.content, row.summary || '', row.id]
|
||
);
|
||
}
|
||
for (const row of deletes) {
|
||
await conn.execute('DELETE FROM rules WHERE id = ?', [row.id]);
|
||
}
|
||
await conn.commit();
|
||
} catch (error) {
|
||
await conn.rollback();
|
||
throw error;
|
||
} finally {
|
||
conn.release();
|
||
}
|
||
}
|
||
|
||
setTimeout(async () => { await pool.end(); }, 250);
|
||
}
|
||
|
||
main().catch(async error => {
|
||
console.error(error);
|
||
try { await pool.end(); } catch {}
|
||
process.exit(1);
|
||
});
|