dwroller/scripts/tag-and-dedup-rules.js

#!/usr/bin/env node
/**
 * tag-and-dedup-rules.js
 *
 * 1. Re-tags "sanitized" rules with their real source (Core Rulebook).
 * 2. Deduplicates rules with the same normalised title, keeping the
 *    version from the most authoritative source.
 *
 * Source priority (highest first):
 *   Core Rulebook > First Founding > Rites of Battle >
 *   Mark of the Xenos > Honour the Chapter > fandom > sanitized
 *
 * Usage:
 *   node scripts/tag-and-dedup-rules.js
 *   node scripts/tag-and-dedup-rules.js --dry-run   (print, don't save)
 */

const { rulesHelpers, logToFile } = require('../database/mariadb');
const mysql = require('mysql2/promise');

const DRY_RUN = process.argv.includes('--dry-run');

const SOURCE_PRIORITY = [
  'Core Rulebook',
  'First Founding',
  'Rites of Battle',
  'Mark of the Xenos',
  'Honour the Chapter',
  'https://40k-rpg-ffg.fandom.com',
  'fandom',
  'sanitized',
];

function priority(source) {
  const i = SOURCE_PRIORITY.indexOf(source);
  return i === -1 ? SOURCE_PRIORITY.length : i;
}

function normalizeTitle(t) {
  return (t || '')
    .toLowerCase()
    .replace(/[^a-z0-9 ]/g, ' ')
    .replace(/\s+/g, ' ')
    .trim();
}

async function getPool() {
  if (!process.env.DB_PASSWORD) {
    throw new Error('DB_PASSWORD must be set in database/.env or .env');
  }
  return mysql.createPool({
    host: process.env.DB_HOST || '192.168.1.113',
    port: Number(process.env.DB_PORT || 3307),
    user: process.env.DB_USER || 'deathwatch',
    password: process.env.DB_PASSWORD,
    database: process.env.DB_NAME || 'deathwatch',
    waitForConnections: true,
    connectionLimit: 5,
  });
}

async function main() {
  const pool = await getPool();

  // ── 1. Load all rules ────────────────────────────────────────────────────
  console.log('Loading all rules…');
  const [rows] = await pool.execute('SELECT id, rule_id, title, source, source_abbr, category, LENGTH(content) as clen FROM rules');
  console.log(`  ${rows.length} total rules`);

  // ── 2. Re-tag sanitized → Core Rulebook ──────────────────────────────────
  const sanitized = rows.filter(r => r.source === 'sanitized');
  console.log(`\n[1] Re-tagging ${sanitized.length} "sanitized" rules → Core Rulebook`);

  if (!DRY_RUN && sanitized.length > 0) {
    await pool.execute(
      `UPDATE rules SET source='Core Rulebook', source_abbr='CR' WHERE source='sanitized'`
    );
    console.log(`    ✓ Updated ${sanitized.length} rows`);
    // Refresh
    sanitized.forEach(r => { r.source = 'Core Rulebook'; r.source_abbr = 'CR'; });
  }

  // ── 3. Find duplicates by normalised title ───────────────────────────────
  console.log('\n[2] Detecting duplicate titles…');

  const byNormTitle = {};
  for (const r of rows) {
    const key = normalizeTitle(r.title);
    if (!key) continue;
    if (!byNormTitle[key]) byNormTitle[key] = [];
    // Update source after re-tag
    if (r.source === 'sanitized') r.source = 'Core Rulebook';
    byNormTitle[key].push(r);
  }

  const dupGroups = Object.values(byNormTitle).filter(g => g.length > 1);
  console.log(`  Found ${dupGroups.length} duplicate groups`);

  let deleted = 0;
  const toDelete = [];

  for (const group of dupGroups) {
    // Sort by priority (best source first), then by content length (longer = better)
    group.sort((a, b) => {
      const pd = priority(a.source) - priority(b.source);
      return pd !== 0 ? pd : b.clen - a.clen;
    });

    const keep   = group[0];
    const remove = group.slice(1);

    if (DRY_RUN && deleted < 20) {
      console.log(`  KEEP  [${keep.source}] "${keep.title}" (${keep.clen} chars)`);
      remove.forEach(r => console.log(`  DEL   [${r.source}] "${r.title}" (${r.clen} chars)`));
    }

    remove.forEach(r => toDelete.push(r.id));
    deleted += remove.length;
  }

  console.log(`  Will delete ${toDelete.length} duplicates`);

  if (!DRY_RUN && toDelete.length > 0) {
    // Delete in batches of 500
    for (let i = 0; i < toDelete.length; i += 500) {
      const batch = toDelete.slice(i, i + 500);
      const placeholders = batch.map(() => '?').join(',');
      await pool.execute(`DELETE FROM rules WHERE id IN (${placeholders})`, batch);
    }
    console.log(`  ✓ Deleted ${toDelete.length} duplicate rules`);
  }

  // ── 4. Fix bad categories on fandom/url sources ──────────────────────────
  console.log('\n[3] Fixing source labels for fandom/url entries…');
  if (!DRY_RUN) {
    const [r1] = await pool.execute(
      `UPDATE rules SET source='Core Rulebook', source_abbr='CR' WHERE source LIKE '%fandom%' OR source LIKE '%40k-rpg%'`
    );
    console.log(`  ✓ Fixed ${r1.affectedRows} fandom entries`);
  }

  // ── 5. Final stats ───────────────────────────────────────────────────────
  const [stats] = await pool.execute(`
    SELECT source, COUNT(*) as cnt FROM rules GROUP BY source ORDER BY cnt DESC
  `);
  console.log('\n── Final source distribution:');
  for (const s of stats) {
    console.log(`  ${String(s.cnt).padStart(5)}  ${s.source}`);
  }

  const [total] = await pool.execute('SELECT COUNT(*) as cnt FROM rules');
  console.log(`\nTotal: ${total[0].cnt} rules`);

  await pool.end();
}

main().catch(e => { console.error(e); process.exit(1); });