Fix rules search ranking and import clean skills/talents from Core Rulebook

- Add search relevance scoring: exact title > title prefix > title contains > Core Rulebook source > content match
- Add DB_PASSWORD env var to pm2.config.js so server can connect to MariaDB
- Add update-rules-from-sources.py: imports 45 skills from CSV and 44 talents from CR.txt, removes garbage advance-table entries
- Add fix-talent-titles.py: normalises OCR-garbled talent titles, removes 28 duplicates, fixes 65 entries
- Add cleanup-skill-dupes.py: removes duplicate skills from csv-import/fandom/sanitized sources, recategorises 55 lore entries

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

database/pm2.config.js

@@ -7,7 +7,8 @@ module.exports = {
       watch: false,
       env: {
         NODE_ENV: 'production',
-        PORT: 5000
+        PORT: 5000,
+        DB_PASSWORD: 'DwRoller@2025!'
       }
     }
   ]

database/routes/rulesRoutes.js

@@ -97,14 +97,28 @@ router.get('/search', async (req, res) => {
     const allRules = await getAllRules();
     const term = query.toLowerCase();
     
+    const score = (rule, t) => {
+      const title = (rule.title || '').toLowerCase();
+      const content = (rule.content || '').toLowerCase();
+      const exactTitle = title === t;
+      const titleStarts = title.startsWith(t);
+      const titleContains = title.includes(t);
+      const isCanonical = rule.source === 'Core Rulebook';
+      // Higher = shown first
+      return (exactTitle ? 100 : 0) +
+             (titleStarts ? 50 : 0) +
+             (titleContains ? 20 : 0) +
+             (isCanonical ? 10 : 0) +
+             (content.includes(t) ? 1 : 0);
+    };
+
     let filtered = allRules.filter(rule => {
       const titleMatch = rule.title && rule.title.toLowerCase().includes(term);
       const contentMatch = rule.content && rule.content.toLowerCase().includes(term);
       const categoryMatch = !category || category === 'all' || rule.category === category;
-      
       return (titleMatch || contentMatch) && categoryMatch;
     });
-    
+
     // If category filtering yielded no results, try without category filter
     if (filtered.length === 0 && category && category !== 'all') {
       filtered = allRules.filter(rule => {
@@ -113,7 +127,10 @@ router.get('/search', async (req, res) => {
         return titleMatch || contentMatch;
       });
     }
-    
+
+    // Sort: title matches first, then Core Rulebook source, then content matches
+    filtered.sort((a, b) => score(b, term) - score(a, term));
+
     // Limit results and truncate content
     const results = filtered.slice(0, limitInt).map(r => ({
       ...r,

scripts/cleanup-skill-dupes.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+Remove duplicate/inferior skill entries now that we have clean Core Rulebook versions.
+Also recategorize clearly non-skill sanitized entries.
+"""
+import re, mysql.connector
+
+DB = dict(host='192.168.1.113', port=3307, user='deathwatch', password='DwRoller@2025!', database='deathwatch')
+db = mysql.connector.connect(**DB)
+c = db.cursor()
+
+# 1. Get all clean skill names (Core Rulebook source)
+c.execute("SELECT title FROM rules WHERE source='Core Rulebook' AND category='skills'")
+clean_skill_names = {row[0].lower() for row in c.fetchall()}
+print(f"Clean skills: {len(clean_skill_names)}")
+
+# 2. Delete old csv-import entries for skills we now have clean versions of
+c.execute("SELECT id, title FROM rules WHERE source='csv-import' AND category IN ('skills','Skills/Rules')")
+csv_skills = c.fetchall()
+deleted_csv = 0
+for db_id, title in csv_skills:
+    if title.lower() in clean_skill_names:
+        c.execute("DELETE FROM rules WHERE id=%s", (db_id,))
+        deleted_csv += 1
+print(f"Deleted {deleted_csv} old csv-import skill duplicates")
+
+# 3. Delete old fandom entries for skills we now have clean versions of
+c.execute("SELECT id, title FROM rules WHERE source IN ('fandom','https://40k-rpg-ffg.fandom.com') AND category IN ('skills','Skills/Rules')")
+fandom_skills = c.fetchall()
+deleted_fandom = 0
+for db_id, title in fandom_skills:
+    if title.lower() in clean_skill_names:
+        c.execute("DELETE FROM rules WHERE id=%s", (db_id,))
+        deleted_fandom += 1
+print(f"Deleted {deleted_fandom} old fandom skill duplicates")
+
+# 4. Delete sanitized duplicate skill entries (where we have an exact title match in Core Rulebook)
+c.execute("SELECT id, title FROM rules WHERE source='sanitized' AND category='skills'")
+sanitized_skills = c.fetchall()
+deleted_san = 0
+for db_id, title in sanitized_skills:
+    if title.lower() in clean_skill_names:
+        c.execute("DELETE FROM rules WHERE id=%s", (db_id,))
+        deleted_san += 1
+print(f"Deleted {deleted_san} sanitized skill duplicates")
+
+db.commit()
+
+# 5. Recategorize sanitized "skills" that are clearly lore/background
+# Rules whose titles suggest they're lore, not skills
+lore_patterns = [
+    'Chapter Recruitment', 'Dark Heresy', 'Watch Fortress', 'Initiations',
+    'Combat Doctrine', 'THE AGES', 'Hive Fleet', 'Blood Trinity',
+    'Military Machine', 'Recruitment', 'Castobel', 'Warp', 'Erioch',
+    'Jericho', 'Tau', 'Tyranid', 'Ork', 'Eldar', 'Necron', 'Chaos',
+]
+c.execute("SELECT id, title FROM rules WHERE source='sanitized' AND category='skills'")
+sanitized_remaining = c.fetchall()
+recatd = 0
+for db_id, title in sanitized_remaining:
+    for pat in lore_patterns:
+        if pat.lower() in title.lower():
+            c.execute("UPDATE rules SET category='lore' WHERE id=%s", (db_id,))
+            recatd += 1
+            break
+print(f"Recategorized {recatd} lore entries from 'skills' to 'lore'")
+
+# 6. Rules-category cleanup: remove duplicate numeric/fragment titles
+c.execute("DELETE FROM rules WHERE source='sanitized' AND category='skills' AND title REGEXP '^[0-9]+$'")
+print(f"Deleted {c.rowcount} numeric-title entries")
+
+db.commit()
+
+# Final stats
+c.execute("SELECT category, COUNT(*) FROM rules GROUP BY category ORDER BY COUNT(*) DESC")
+print("\n=== Rules by category ===")
+for cat, cnt in c.fetchall():
+    print(f"  {cat}: {cnt}")
+
+c.execute("SELECT source, COUNT(*) FROM rules GROUP BY source ORDER BY COUNT(*) DESC")
+print("\n=== Rules by source ===")
+total = 0
+for src, cnt in c.fetchall():
+    print(f"  {src}: {cnt}")
+    total += cnt
+print(f"  TOTAL: {total}")
+
+c.close()
+db.close()

scripts/fix-talent-titles.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""
+Fix garbled OCR talent titles in the rules DB.
+- Normalise mixed-case OCR titles to proper Title Case
+- Set source/source_abbr/category correctly for all talent entries
+- Remove duplicate sanitized entries where a clean 'talent_*' version exists
+"""
+import re, mysql.connector
+
+DB = dict(host='192.168.1.113', port=3307, user='deathwatch',
+          password='DwRoller@2025!', database='deathwatch')
+
+def normalise_title(raw):
+    """Convert OCR-garbled title to proper Title Case."""
+    # Remove excess whitespace
+    s = re.sub(r'\s+', ' ', raw).strip()
+    # Lowercase everything first, then title-case
+    words = s.lower().split()
+    # Keep small words lowercase unless first word
+    small = {'a','an','the','of','to','in','for','and','or','but','with','at','by','from'}
+    result = []
+    for i, w in enumerate(words):
+        if i == 0 or w not in small:
+            # Handle hyphenated words
+            result.append('-'.join(p.capitalize() for p in w.split('-')))
+        else:
+            result.append(w)
+    return ' '.join(result)
+
+db = mysql.connector.connect(**DB)
+c = db.cursor()
+
+# 1. Get all sanitized talent entries (have "Prerequisites:" in content)
+c.execute("""SELECT id, rule_id, title FROM rules
+             WHERE source = 'sanitized' AND content LIKE 'Prerequisites:%'""")
+sanitized_talents = c.fetchall()
+print(f"Found {len(sanitized_talents)} sanitized talent entries")
+
+# 2. Get existing clean talent rule_ids (from our earlier import)
+c.execute("SELECT rule_id, title FROM rules WHERE source = 'Core Rulebook' AND category = 'talents'")
+clean_talents = {row[0]: row[1] for row in c.fetchall()}
+print(f"Found {len(clean_talents)} clean talent entries")
+
+fixed = 0
+deleted = 0
+
+for (db_id, rule_id, title) in sanitized_talents:
+    clean_title = normalise_title(title)
+    expected_rid = 'talent_' + re.sub(r'[^a-z0-9]', '_', clean_title.lower())
+    # Remove trailing underscores
+    expected_rid = re.sub(r'_+$', '', expected_rid)
+    expected_rid = re.sub(r'_+', '_', expected_rid)
+
+    if expected_rid in clean_talents:
+        # We already have a clean version - delete the sanitized duplicate
+        c.execute("DELETE FROM rules WHERE id = %s", (db_id,))
+        deleted += 1
+    else:
+        # No clean version exists - fix this entry's title/source/category
+        c.execute("""UPDATE rules
+                     SET title = %s, source = 'Core Rulebook', source_abbr = 'CR', category = 'talents'
+                     WHERE id = %s""",
+                  (clean_title, db_id))
+        # Also add a proper rule_id if missing
+        new_rid = expected_rid
+        # Check if rule_id already conflicts
+        c.execute("SELECT COUNT(*) FROM rules WHERE rule_id = %s AND id != %s", (new_rid, db_id))
+        if c.fetchone()[0] == 0:
+            c.execute("UPDATE rules SET rule_id = %s WHERE id = %s", (new_rid, db_id))
+        fixed += 1
+
+db.commit()
+print(f"Fixed: {fixed} talent titles, Deleted: {deleted} duplicates")
+
+# 3. Also fix remaining garbled-title talents (non-sanitized, left-over OCR artifacts)
+c.execute("""SELECT id, title FROM rules
+             WHERE category = 'talents' AND title REGEXP '[A-Z]{2,}[a-z]' """)
+garbled = c.fetchall()
+print(f"\nFixing {len(garbled)} remaining garbled-title talent entries...")
+for (db_id, title) in garbled:
+    clean = normalise_title(title)
+    if clean != title:
+        c.execute("UPDATE rules SET title = %s WHERE id = %s", (clean, db_id))
+        print(f"  '{title}' → '{clean}'")
+
+db.commit()
+
+# 4. Final stats
+c.execute("SELECT source, COUNT(*) FROM rules GROUP BY source ORDER BY COUNT(*) DESC")
+print("\n=== Final rule counts by source ===")
+total = 0
+for src, cnt in c.fetchall():
+    print(f"  {src}: {cnt}")
+    total += cnt
+print(f"  TOTAL: {total}")
+
+c.execute("SELECT category, COUNT(*) FROM rules GROUP BY category ORDER BY COUNT(*) DESC")
+print("\n=== By category ===")
+for cat, cnt in c.fetchall():
+    print(f"  {cat}: {cnt}")
+
+c.close()
+db.close()

scripts/update-rules-from-sources.py

@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+"""
+Update rules database from authoritative sources:
+  1. Skills  → database/deathwatch_skills_p94_107.csv
+  2. Talents → database/rules/CR.txt  (Chapter IV, lines ~7272-8431)
+  3. Delete garbled advance-table entries
+
+Uses the REST API at http://localhost:5000
+"""
+
+import csv, json, re, sys, urllib.request, urllib.error
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+API  = "http://localhost:5000/api"
+GM_SECRET = "bongo"
+
+# ── helpers ───────────────────────────────────────────────────────────────────
+
+def api_call(method, path, body=None):
+    url = API + path
+    data = json.dumps(body).encode() if body else None
+    headers = {"Content-Type": "application/json", "x-gm-secret": GM_SECRET}
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+    try:
+        with urllib.request.urlopen(req, timeout=15) as r:
+            return json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        print(f"  HTTP {e.code} {method} {path}: {e.read().decode()[:200]}")
+        return None
+
+def upsert_rule(rule_id, title, content, page, source, source_abbr, category):
+    """Insert or replace a rule in the DB via direct SQL through a helper endpoint.
+    Falls back to delete+insert since the API doesn't expose a PATCH endpoint."""
+    import mysql.connector
+    # We'll do this via mysql directly (faster than HTTP for bulk)
+    return (rule_id, title, content, page, source, source_abbr, category)
+
+# ── 1. Skills from CSV ─────────────────────────────────────────────────────────
+
+def load_skills():
+    csvpath = REPO / "database" / "deathwatch_skills_p94_107.csv"
+    with open(csvpath, newline="", encoding="utf-8") as f:
+        return list(csv.DictReader(f))
+
+def build_skill_content(row):
+    parts = []
+    if row.get("skill_text"):
+        parts.append(row["skill_text"].strip())
+    if row.get("skill_description"):
+        parts.append(row["skill_description"].strip())
+    if row.get("skill_use"):
+        parts.append(f"Use: {row['skill_use'].strip()}")
+    return "\n\n".join(parts)
+
+# ── 2. Talents from CR.txt ─────────────────────────────────────────────────────
+
+# Known talent names (from Table 4-1, p108-111 of Core Rulebook)
+KNOWN_TALENTS = {
+    "abhor the witch", "air of authority", "ambidextrous",
+    "assassin strike", "astartes weapon specialisation",
+    "battle rage", "berserk charge", "binary chatter",
+    "blademaster", "blind fighting", "bulging biceps",
+    "catfall", "combat formation", "combat master",
+    "crack shot", "crushing blow", "deadeye shot",
+    "die hard", "disturbing voice", "double team",
+    "dual shot", "dual strike", "enemy", "eye of vengeance",
+    "fast hands", "fearless", "foresight",
+    "gunslinger", "hard target", "hatred",
+    "heightened senses", "hip shooting", "horde fighter",
+    "hunter of aliens", "independent targeting",
+    "inspire wrath", "into the jaws of hell",
+    "iron affinity", "iron discipline", "iron jaw",
+    "jaded", "killing strike", "leap up",
+    "leaping dodge", "light sleeper", "lightning attack",
+    "lightning reflexes", "litany of hate", "logis implant",
+    "luminen blast", "luminen charge", "luminen shock",
+    "maglev grace", "marksman", "master craftsman",
+    "mechadendrite use", "meditation", "mental rage",
+    "mighty shot", "nerves of steel", "night vision",
+    "peer", "pistol training", "precise blow",
+    "psy rating", "quick draw", "rapid reaction",
+    "rapid reload", "resistance", "rite of fear",
+    "rite of pure thought", "sacrificial strike",
+    "sharpshooter", "slayer of daemons", "sound constitution",
+    "sprint", "step aside", "storm of iron",
+    "strong minded", "swift attack", "talent",
+    "thunder charge", "true grit", "two weapon wielder",
+    "unarmed master", "unarmed warrior",
+    "unnatural characteristic", "warp affinity",
+    "weapon training",
+    # Chapter speciality talents
+    "astartes assault", "astartes devastator", "astartes librarian",
+    "astartes tactical", "astartes techmarine",
+    "basic weapon training", "bolt weapon mastery",
+    "energy cache", "enhanced bionic frame",
+    "frenzy", "furious assault", "flesh render",
+    "grenade mastery", "hammer blow", "hard target",
+    "head shot", "incendiary charge", "mechadendrite use",
+    "overcharge", "overwatch", "precision killer",
+    "stay vigilant", "tactical advance",
+}
+
+def normalise_talent_name(raw):
+    """Normalise OCR talent name to lowercase clean string."""
+    # Remove OCR random-cap artifacts: if string is mixed case with no obvious pattern,
+    # normalise to lowercase
+    return re.sub(r'\s+', ' ', raw).strip().lower()
+
+def split_two_columns(line):
+    """Split a line from a two-column PDF layout.
+    Returns (left, right) stripped strings, either may be empty."""
+    # Find the largest run of whitespace (gap between columns)
+    matches = list(re.finditer(r' {5,}', line))
+    if not matches:
+        return line.strip(), ""
+    # Pick the widest gap; if there are multiple, prefer the one closest to centre
+    best = max(matches, key=lambda m: m.end() - m.start())
+    left  = line[:best.start()].strip()
+    right = line[best.end():].strip()
+    return left, right
+
+def looks_like_talent_name(s):
+    """Heuristic: a talent name is a short phrase (≤8 words) with no digits/punctuation."""
+    s = s.strip()
+    if not s or len(s) > 70 or len(s) < 3:
+        return False
+    if re.search(r'[0-9:;|]', s):
+        return False
+    words = s.split()
+    if len(words) > 8:
+        return False
+    # Must be mostly letters
+    letters = sum(c.isalpha() for c in s)
+    if letters / len(s) < 0.7:
+        return False
+    norm = normalise_talent_name(s)
+    # Must match a known talent or be a plausible name
+    for kt in KNOWN_TALENTS:
+        if kt == norm or kt.startswith(norm) or norm.startswith(kt):
+            return True
+    # fallback: allow if it looks like a heading (no verb, short)
+    return False
+
+def parse_talents_from_cr(cr_path):
+    """Parse the talent descriptions from CR.txt using column-splitting."""
+    with open(cr_path, encoding="utf-8") as f:
+        lines = f.readlines()
+
+    # Find start of talent descriptions (after the talent table)
+    START_HINT = 7272   # from analysis
+    END_HINT   = 8450
+
+    relevant = lines[START_HINT - 1 : END_HINT]
+
+    # Rebuild two separate column streams
+    left_stream  = []
+    right_stream = []
+
+    for line in relevant:
+        line = line.rstrip('\n')
+        l, r = split_two_columns(line)
+        left_stream.append(l)
+        right_stream.append(r)
+
+    def extract_talents_from_stream(stream):
+        """Extract {name, prerequisites, description} dicts from one column stream."""
+        talents = []
+        current = None
+        prereq_done = False
+
+        for line in stream:
+            line = line.strip()
+            if not line:
+                continue
+
+            # Check if this is a "Prerequisites:" line
+            pm = re.match(r'^Prerequisites?:\s*(.*)', line, re.IGNORECASE)
+            if pm and current is not None:
+                current['prerequisites'] = pm.group(1).strip()
+                prereq_done = True
+                continue
+
+            # Check if it looks like a talent name
+            if looks_like_talent_name(line) and (current is None or prereq_done):
+                if current and current.get('description'):
+                    talents.append(current)
+                current = {'name': line, 'prerequisites': '—', 'description': ''}
+                prereq_done = False
+                continue
+
+            # Otherwise it's description text
+            if current is not None:
+                if current['description']:
+                    current['description'] += ' ' + line
+                else:
+                    current['description'] = line
+
+        if current and current.get('description'):
+            talents.append(current)
+
+        return talents
+
+    left_talents  = extract_talents_from_stream(left_stream)
+    right_talents = extract_talents_from_stream(right_stream)
+
+    # Merge and deduplicate by normalised name
+    all_talents = {}
+    for t in left_talents + right_talents:
+        norm = normalise_talent_name(t['name'])
+        if norm not in all_talents or len(t['description']) > len(all_talents[norm]['description']):
+            all_talents[norm] = t
+
+    return list(all_talents.values())
+
+# ── 3. Direct DB update via mysql.connector ────────────────────────────────────
+
+def get_db():
+    import mysql.connector
+    return mysql.connector.connect(
+        host="192.168.1.113", port=3307,
+        user="deathwatch",
+        password=open(REPO / ".env.db").read().strip() if (REPO / ".env.db").exists() else "DwRoller@2025!",
+        database="deathwatch"
+    )
+
+def run():
+    print("=== Deathwatch Rules Updater ===\n")
+
+    # Try direct DB connection
+    try:
+        db = get_db()
+        cursor = db.cursor()
+        print("Connected to MariaDB directly.\n")
+    except Exception as e:
+        print(f"Cannot connect to MariaDB: {e}")
+        print("Falling back to HTTP API (slower).")
+        db = None
+        cursor = None
+
+    def exec_sql(sql, params=()):
+        if cursor:
+            cursor.execute(sql, params)
+        else:
+            print(f"  [no-db] would execute: {sql[:80]}")
+
+    def upsert(rule_id, title, content, page, source, source_abbr, category):
+        if cursor:
+            cursor.execute(
+                """INSERT INTO rules (rule_id, title, content, page, source, source_abbr, category)
+                   VALUES (%s, %s, %s, %s, %s, %s, %s)
+                   ON DUPLICATE KEY UPDATE
+                     title=VALUES(title), content=VALUES(content),
+                     page=VALUES(page), source=VALUES(source),
+                     source_abbr=VALUES(source_abbr), category=VALUES(category)""",
+                (rule_id, title, content, page, source, source_abbr, category)
+            )
+        else:
+            print(f"  [no-db] upsert '{title}'")
+
+    # ── Step 1: Delete garbage advance-table entries ──────────────────────────
+    print("Step 1: Removing garbled advance-table entries...")
+    garbage_patterns = [
+        # Advance tables masquerading as rules
+        "title LIKE 'AdvanceCostType%'",
+        # Rules whose entire content is a cost/prerequisites table
+        "content REGEXP '^[A-Za-z ]+[[:space:]]+[0-9,]+[[:space:]]+(Skill|Talent)[[:space:]]'",
+        # Very short meaningless entries (under 30 chars content, sanitized source)
+        "(source = 'sanitized' AND LENGTH(content) < 40)",
+    ]
+    deleted = 0
+    for pat in garbage_patterns:
+        if cursor:
+            cursor.execute(f"SELECT COUNT(*) FROM rules WHERE {pat}")
+            count = cursor.fetchone()[0]
+            cursor.execute(f"DELETE FROM rules WHERE {pat}")
+            print(f"  Deleted {count} rows matching: {pat[:60]}")
+            deleted += count
+        else:
+            print(f"  [no-db] would delete: {pat[:60]}")
+    print(f"  Total deleted: {deleted}\n")
+
+    # ── Step 2: Skills from CSV ───────────────────────────────────────────────
+    print("Step 2: Upserting skills from CSV...")
+    skills = load_skills()
+    for skill in skills:
+        name    = skill['name'].strip()
+        content = build_skill_content(skill)
+        char    = skill.get('characteristic', '').strip()
+        stype   = skill.get('type', '').strip()
+        desc_header = f"{stype} Skill – {char}" if char else stype
+        full_content = f"{desc_header}\n\n{content}" if desc_header else content
+
+        rule_id = "skill_" + re.sub(r'[^a-z0-9]', '_', name.lower())
+        upsert(rule_id, name, full_content, None,
+               "Core Rulebook", "CR", "skills")
+        print(f"  ✓ {name}")
+    if db:
+        db.commit()
+    print(f"  {len(skills)} skills updated.\n")
+
+    # ── Step 3: Talents from CR.txt ───────────────────────────────────────────
+    print("Step 3: Parsing and upserting talents from CR.txt...")
+    cr_path = REPO / "database" / "rules" / "CR.txt"
+    talents = parse_talents_from_cr(cr_path)
+    print(f"  Parsed {len(talents)} talent entries from CR.txt")
+
+    imported = 0
+    for t in talents:
+        name = re.sub(r'\s+', ' ', t['name']).strip()
+        # Clean up OCR capitalisation: convert random-caps to title case
+        name_clean = re.sub(r'([A-Z])', lambda m: m.group(1).lower(), name)
+        name_clean = ' '.join(w.capitalize() for w in name_clean.split())
+
+        prereq = t.get('prerequisites', '—').strip()
+        desc   = t.get('description', '').strip()
+        if len(desc) < 20:
+            continue  # Skip empty/garbage entries
+
+        content = f"Prerequisites: {prereq}\n\n{desc}"
+        rule_id = "talent_" + re.sub(r'[^a-z0-9]', '_', name_clean.lower())
+
+        upsert(rule_id, name_clean, content, None,
+               "Core Rulebook", "CR", "talents")
+        imported += 1
+
+    if db:
+        db.commit()
+    print(f"  {imported} talents upserted.\n")
+
+    # ── Step 4: Fix existing sanitized rules – remove pure table entries ──────
+    print("Step 4: Cleaning up remaining garbled sanitized entries...")
+    if cursor:
+        # Rules where the title IS the first line of the content (typical for advance tables)
+        cursor.execute("""
+            DELETE FROM rules
+            WHERE source = 'sanitized'
+              AND (
+                content LIKE '%Cost%Type%Prerequisites%'
+                OR content REGEXP '[0-9]{3,}[[:space:]]+(Skill|Talent)[[:space:]]'
+                OR (title = content)
+              )
+        """)
+        n = cursor.rowcount
+        db.commit()
+        print(f"  Deleted {n} more garbled entries.\n")
+    else:
+        print("  [no-db] skipped.\n")
+
+    # ── Done ──────────────────────────────────────────────────────────────────
+    if cursor:
+        cursor.execute("SELECT COUNT(*) FROM rules")
+        total = cursor.fetchone()[0]
+        cursor.execute("SELECT source, COUNT(*) FROM rules GROUP BY source ORDER BY COUNT(*) DESC")
+        by_source = cursor.fetchall()
+        print(f"=== Done. Total rules in DB: {total} ===")
+        for src, cnt in by_source:
+            print(f"  {src}: {cnt}")
+        cursor.close()
+        db.close()
+    else:
+        print("=== Done (dry-run, no DB connection). ===")
+
+if __name__ == "__main__":
+    run()