Alex
1e361301ad
- Add search relevance scoring: exact title > title prefix > title contains > Core Rulebook source > content match - Add DB_PASSWORD env var to pm2.config.js so server can connect to MariaDB - Add update-rules-from-sources.py: imports 45 skills from CSV and 44 talents from CR.txt, removes garbage advance-table entries - Add fix-talent-titles.py: normalises OCR-garbled talent titles, removes 28 duplicates, fixes 65 entries - Add cleanup-skill-dupes.py: removes duplicate skills from csv-import/fandom/sanitized sources, recategorises 55 lore entries Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
90 lines
3.5 KiB
Python
90 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Remove duplicate/inferior skill entries now that we have clean Core Rulebook versions.
|
|
Also recategorize clearly non-skill sanitized entries.
|
|
"""
|
|
import re, mysql.connector
|
|
|
|
DB = dict(host='192.168.1.113', port=3307, user='deathwatch', password='DwRoller@2025!', database='deathwatch')
|
|
db = mysql.connector.connect(**DB)
|
|
c = db.cursor()
|
|
|
|
# 1. Get all clean skill names (Core Rulebook source)
|
|
c.execute("SELECT title FROM rules WHERE source='Core Rulebook' AND category='skills'")
|
|
clean_skill_names = {row[0].lower() for row in c.fetchall()}
|
|
print(f"Clean skills: {len(clean_skill_names)}")
|
|
|
|
# 2. Delete old csv-import entries for skills we now have clean versions of
|
|
c.execute("SELECT id, title FROM rules WHERE source='csv-import' AND category IN ('skills','Skills/Rules')")
|
|
csv_skills = c.fetchall()
|
|
deleted_csv = 0
|
|
for db_id, title in csv_skills:
|
|
if title.lower() in clean_skill_names:
|
|
c.execute("DELETE FROM rules WHERE id=%s", (db_id,))
|
|
deleted_csv += 1
|
|
print(f"Deleted {deleted_csv} old csv-import skill duplicates")
|
|
|
|
# 3. Delete old fandom entries for skills we now have clean versions of
|
|
c.execute("SELECT id, title FROM rules WHERE source IN ('fandom','https://40k-rpg-ffg.fandom.com') AND category IN ('skills','Skills/Rules')")
|
|
fandom_skills = c.fetchall()
|
|
deleted_fandom = 0
|
|
for db_id, title in fandom_skills:
|
|
if title.lower() in clean_skill_names:
|
|
c.execute("DELETE FROM rules WHERE id=%s", (db_id,))
|
|
deleted_fandom += 1
|
|
print(f"Deleted {deleted_fandom} old fandom skill duplicates")
|
|
|
|
# 4. Delete sanitized duplicate skill entries (where we have an exact title match in Core Rulebook)
|
|
c.execute("SELECT id, title FROM rules WHERE source='sanitized' AND category='skills'")
|
|
sanitized_skills = c.fetchall()
|
|
deleted_san = 0
|
|
for db_id, title in sanitized_skills:
|
|
if title.lower() in clean_skill_names:
|
|
c.execute("DELETE FROM rules WHERE id=%s", (db_id,))
|
|
deleted_san += 1
|
|
print(f"Deleted {deleted_san} sanitized skill duplicates")
|
|
|
|
db.commit()
|
|
|
|
# 5. Recategorize sanitized "skills" that are clearly lore/background
|
|
# Rules whose titles suggest they're lore, not skills
|
|
lore_patterns = [
|
|
'Chapter Recruitment', 'Dark Heresy', 'Watch Fortress', 'Initiations',
|
|
'Combat Doctrine', 'THE AGES', 'Hive Fleet', 'Blood Trinity',
|
|
'Military Machine', 'Recruitment', 'Castobel', 'Warp', 'Erioch',
|
|
'Jericho', 'Tau', 'Tyranid', 'Ork', 'Eldar', 'Necron', 'Chaos',
|
|
]
|
|
c.execute("SELECT id, title FROM rules WHERE source='sanitized' AND category='skills'")
|
|
sanitized_remaining = c.fetchall()
|
|
recatd = 0
|
|
for db_id, title in sanitized_remaining:
|
|
for pat in lore_patterns:
|
|
if pat.lower() in title.lower():
|
|
c.execute("UPDATE rules SET category='lore' WHERE id=%s", (db_id,))
|
|
recatd += 1
|
|
break
|
|
print(f"Recategorized {recatd} lore entries from 'skills' to 'lore'")
|
|
|
|
# 6. Rules-category cleanup: remove duplicate numeric/fragment titles
|
|
c.execute("DELETE FROM rules WHERE source='sanitized' AND category='skills' AND title REGEXP '^[0-9]+$'")
|
|
print(f"Deleted {c.rowcount} numeric-title entries")
|
|
|
|
db.commit()
|
|
|
|
# Final stats
|
|
c.execute("SELECT category, COUNT(*) FROM rules GROUP BY category ORDER BY COUNT(*) DESC")
|
|
print("\n=== Rules by category ===")
|
|
for cat, cnt in c.fetchall():
|
|
print(f" {cat}: {cnt}")
|
|
|
|
c.execute("SELECT source, COUNT(*) FROM rules GROUP BY source ORDER BY COUNT(*) DESC")
|
|
print("\n=== Rules by source ===")
|
|
total = 0
|
|
for src, cnt in c.fetchall():
|
|
print(f" {src}: {cnt}")
|
|
total += cnt
|
|
print(f" TOTAL: {total}")
|
|
|
|
c.close()
|
|
db.close()
|