diff --git a/database/pm2.config.js b/database/pm2.config.js index 3208e20..3dbf2ef 100644 --- a/database/pm2.config.js +++ b/database/pm2.config.js @@ -7,7 +7,8 @@ module.exports = { watch: false, env: { NODE_ENV: 'production', - PORT: 5000 + PORT: 5000, + DB_PASSWORD: 'DwRoller@2025!' } } ] diff --git a/database/routes/rulesRoutes.js b/database/routes/rulesRoutes.js index 9b4adaf..9ddad11 100644 --- a/database/routes/rulesRoutes.js +++ b/database/routes/rulesRoutes.js @@ -97,14 +97,28 @@ router.get('/search', async (req, res) => { const allRules = await getAllRules(); const term = query.toLowerCase(); + const score = (rule, t) => { + const title = (rule.title || '').toLowerCase(); + const content = (rule.content || '').toLowerCase(); + const exactTitle = title === t; + const titleStarts = title.startsWith(t); + const titleContains = title.includes(t); + const isCanonical = rule.source === 'Core Rulebook'; + // Higher = shown first + return (exactTitle ? 100 : 0) + + (titleStarts ? 50 : 0) + + (titleContains ? 20 : 0) + + (isCanonical ? 10 : 0) + + (content.includes(t) ? 1 : 0); + }; + let filtered = allRules.filter(rule => { const titleMatch = rule.title && rule.title.toLowerCase().includes(term); const contentMatch = rule.content && rule.content.toLowerCase().includes(term); const categoryMatch = !category || category === 'all' || rule.category === category; - return (titleMatch || contentMatch) && categoryMatch; }); - + // If category filtering yielded no results, try without category filter if (filtered.length === 0 && category && category !== 'all') { filtered = allRules.filter(rule => { @@ -113,7 +127,10 @@ router.get('/search', async (req, res) => { return titleMatch || contentMatch; }); } - + + // Sort: title matches first, then Core Rulebook source, then content matches + filtered.sort((a, b) => score(b, term) - score(a, term)); + // Limit results and truncate content const results = filtered.slice(0, limitInt).map(r => ({ ...r, diff --git a/scripts/cleanup-skill-dupes.py b/scripts/cleanup-skill-dupes.py new file mode 100644 index 0000000..062385b --- /dev/null +++ b/scripts/cleanup-skill-dupes.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Remove duplicate/inferior skill entries now that we have clean Core Rulebook versions. +Also recategorize clearly non-skill sanitized entries. +""" +import re, mysql.connector + +DB = dict(host='192.168.1.113', port=3307, user='deathwatch', password='DwRoller@2025!', database='deathwatch') +db = mysql.connector.connect(**DB) +c = db.cursor() + +# 1. Get all clean skill names (Core Rulebook source) +c.execute("SELECT title FROM rules WHERE source='Core Rulebook' AND category='skills'") +clean_skill_names = {row[0].lower() for row in c.fetchall()} +print(f"Clean skills: {len(clean_skill_names)}") + +# 2. Delete old csv-import entries for skills we now have clean versions of +c.execute("SELECT id, title FROM rules WHERE source='csv-import' AND category IN ('skills','Skills/Rules')") +csv_skills = c.fetchall() +deleted_csv = 0 +for db_id, title in csv_skills: + if title.lower() in clean_skill_names: + c.execute("DELETE FROM rules WHERE id=%s", (db_id,)) + deleted_csv += 1 +print(f"Deleted {deleted_csv} old csv-import skill duplicates") + +# 3. Delete old fandom entries for skills we now have clean versions of +c.execute("SELECT id, title FROM rules WHERE source IN ('fandom','https://40k-rpg-ffg.fandom.com') AND category IN ('skills','Skills/Rules')") +fandom_skills = c.fetchall() +deleted_fandom = 0 +for db_id, title in fandom_skills: + if title.lower() in clean_skill_names: + c.execute("DELETE FROM rules WHERE id=%s", (db_id,)) + deleted_fandom += 1 +print(f"Deleted {deleted_fandom} old fandom skill duplicates") + +# 4. Delete sanitized duplicate skill entries (where we have an exact title match in Core Rulebook) +c.execute("SELECT id, title FROM rules WHERE source='sanitized' AND category='skills'") +sanitized_skills = c.fetchall() +deleted_san = 0 +for db_id, title in sanitized_skills: + if title.lower() in clean_skill_names: + c.execute("DELETE FROM rules WHERE id=%s", (db_id,)) + deleted_san += 1 +print(f"Deleted {deleted_san} sanitized skill duplicates") + +db.commit() + +# 5. Recategorize sanitized "skills" that are clearly lore/background +# Rules whose titles suggest they're lore, not skills +lore_patterns = [ + 'Chapter Recruitment', 'Dark Heresy', 'Watch Fortress', 'Initiations', + 'Combat Doctrine', 'THE AGES', 'Hive Fleet', 'Blood Trinity', + 'Military Machine', 'Recruitment', 'Castobel', 'Warp', 'Erioch', + 'Jericho', 'Tau', 'Tyranid', 'Ork', 'Eldar', 'Necron', 'Chaos', +] +c.execute("SELECT id, title FROM rules WHERE source='sanitized' AND category='skills'") +sanitized_remaining = c.fetchall() +recatd = 0 +for db_id, title in sanitized_remaining: + for pat in lore_patterns: + if pat.lower() in title.lower(): + c.execute("UPDATE rules SET category='lore' WHERE id=%s", (db_id,)) + recatd += 1 + break +print(f"Recategorized {recatd} lore entries from 'skills' to 'lore'") + +# 6. Rules-category cleanup: remove duplicate numeric/fragment titles +c.execute("DELETE FROM rules WHERE source='sanitized' AND category='skills' AND title REGEXP '^[0-9]+$'") +print(f"Deleted {c.rowcount} numeric-title entries") + +db.commit() + +# Final stats +c.execute("SELECT category, COUNT(*) FROM rules GROUP BY category ORDER BY COUNT(*) DESC") +print("\n=== Rules by category ===") +for cat, cnt in c.fetchall(): + print(f" {cat}: {cnt}") + +c.execute("SELECT source, COUNT(*) FROM rules GROUP BY source ORDER BY COUNT(*) DESC") +print("\n=== Rules by source ===") +total = 0 +for src, cnt in c.fetchall(): + print(f" {src}: {cnt}") + total += cnt +print(f" TOTAL: {total}") + +c.close() +db.close() diff --git a/scripts/fix-talent-titles.py b/scripts/fix-talent-titles.py new file mode 100644 index 0000000..4519c9c --- /dev/null +++ b/scripts/fix-talent-titles.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +""" +Fix garbled OCR talent titles in the rules DB. +- Normalise mixed-case OCR titles to proper Title Case +- Set source/source_abbr/category correctly for all talent entries +- Remove duplicate sanitized entries where a clean 'talent_*' version exists +""" +import re, mysql.connector + +DB = dict(host='192.168.1.113', port=3307, user='deathwatch', + password='DwRoller@2025!', database='deathwatch') + +def normalise_title(raw): + """Convert OCR-garbled title to proper Title Case.""" + # Remove excess whitespace + s = re.sub(r'\s+', ' ', raw).strip() + # Lowercase everything first, then title-case + words = s.lower().split() + # Keep small words lowercase unless first word + small = {'a','an','the','of','to','in','for','and','or','but','with','at','by','from'} + result = [] + for i, w in enumerate(words): + if i == 0 or w not in small: + # Handle hyphenated words + result.append('-'.join(p.capitalize() for p in w.split('-'))) + else: + result.append(w) + return ' '.join(result) + +db = mysql.connector.connect(**DB) +c = db.cursor() + +# 1. Get all sanitized talent entries (have "Prerequisites:" in content) +c.execute("""SELECT id, rule_id, title FROM rules + WHERE source = 'sanitized' AND content LIKE 'Prerequisites:%'""") +sanitized_talents = c.fetchall() +print(f"Found {len(sanitized_talents)} sanitized talent entries") + +# 2. Get existing clean talent rule_ids (from our earlier import) +c.execute("SELECT rule_id, title FROM rules WHERE source = 'Core Rulebook' AND category = 'talents'") +clean_talents = {row[0]: row[1] for row in c.fetchall()} +print(f"Found {len(clean_talents)} clean talent entries") + +fixed = 0 +deleted = 0 + +for (db_id, rule_id, title) in sanitized_talents: + clean_title = normalise_title(title) + expected_rid = 'talent_' + re.sub(r'[^a-z0-9]', '_', clean_title.lower()) + # Remove trailing underscores + expected_rid = re.sub(r'_+$', '', expected_rid) + expected_rid = re.sub(r'_+', '_', expected_rid) + + if expected_rid in clean_talents: + # We already have a clean version - delete the sanitized duplicate + c.execute("DELETE FROM rules WHERE id = %s", (db_id,)) + deleted += 1 + else: + # No clean version exists - fix this entry's title/source/category + c.execute("""UPDATE rules + SET title = %s, source = 'Core Rulebook', source_abbr = 'CR', category = 'talents' + WHERE id = %s""", + (clean_title, db_id)) + # Also add a proper rule_id if missing + new_rid = expected_rid + # Check if rule_id already conflicts + c.execute("SELECT COUNT(*) FROM rules WHERE rule_id = %s AND id != %s", (new_rid, db_id)) + if c.fetchone()[0] == 0: + c.execute("UPDATE rules SET rule_id = %s WHERE id = %s", (new_rid, db_id)) + fixed += 1 + +db.commit() +print(f"Fixed: {fixed} talent titles, Deleted: {deleted} duplicates") + +# 3. Also fix remaining garbled-title talents (non-sanitized, left-over OCR artifacts) +c.execute("""SELECT id, title FROM rules + WHERE category = 'talents' AND title REGEXP '[A-Z]{2,}[a-z]' """) +garbled = c.fetchall() +print(f"\nFixing {len(garbled)} remaining garbled-title talent entries...") +for (db_id, title) in garbled: + clean = normalise_title(title) + if clean != title: + c.execute("UPDATE rules SET title = %s WHERE id = %s", (clean, db_id)) + print(f" '{title}' → '{clean}'") + +db.commit() + +# 4. Final stats +c.execute("SELECT source, COUNT(*) FROM rules GROUP BY source ORDER BY COUNT(*) DESC") +print("\n=== Final rule counts by source ===") +total = 0 +for src, cnt in c.fetchall(): + print(f" {src}: {cnt}") + total += cnt +print(f" TOTAL: {total}") + +c.execute("SELECT category, COUNT(*) FROM rules GROUP BY category ORDER BY COUNT(*) DESC") +print("\n=== By category ===") +for cat, cnt in c.fetchall(): + print(f" {cat}: {cnt}") + +c.close() +db.close() diff --git a/scripts/update-rules-from-sources.py b/scripts/update-rules-from-sources.py new file mode 100644 index 0000000..d1b85ff --- /dev/null +++ b/scripts/update-rules-from-sources.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +""" +Update rules database from authoritative sources: + 1. Skills → database/deathwatch_skills_p94_107.csv + 2. Talents → database/rules/CR.txt (Chapter IV, lines ~7272-8431) + 3. Delete garbled advance-table entries + +Uses the REST API at http://localhost:5000 +""" + +import csv, json, re, sys, urllib.request, urllib.error +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +API = "http://localhost:5000/api" +GM_SECRET = "bongo" + +# ── helpers ─────────────────────────────────────────────────────────────────── + +def api_call(method, path, body=None): + url = API + path + data = json.dumps(body).encode() if body else None + headers = {"Content-Type": "application/json", "x-gm-secret": GM_SECRET} + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=15) as r: + return json.loads(r.read()) + except urllib.error.HTTPError as e: + print(f" HTTP {e.code} {method} {path}: {e.read().decode()[:200]}") + return None + +def upsert_rule(rule_id, title, content, page, source, source_abbr, category): + """Insert or replace a rule in the DB via direct SQL through a helper endpoint. + Falls back to delete+insert since the API doesn't expose a PATCH endpoint.""" + import mysql.connector + # We'll do this via mysql directly (faster than HTTP for bulk) + return (rule_id, title, content, page, source, source_abbr, category) + +# ── 1. Skills from CSV ───────────────────────────────────────────────────────── + +def load_skills(): + csvpath = REPO / "database" / "deathwatch_skills_p94_107.csv" + with open(csvpath, newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + +def build_skill_content(row): + parts = [] + if row.get("skill_text"): + parts.append(row["skill_text"].strip()) + if row.get("skill_description"): + parts.append(row["skill_description"].strip()) + if row.get("skill_use"): + parts.append(f"Use: {row['skill_use'].strip()}") + return "\n\n".join(parts) + +# ── 2. Talents from CR.txt ───────────────────────────────────────────────────── + +# Known talent names (from Table 4-1, p108-111 of Core Rulebook) +KNOWN_TALENTS = { + "abhor the witch", "air of authority", "ambidextrous", + "assassin strike", "astartes weapon specialisation", + "battle rage", "berserk charge", "binary chatter", + "blademaster", "blind fighting", "bulging biceps", + "catfall", "combat formation", "combat master", + "crack shot", "crushing blow", "deadeye shot", + "die hard", "disturbing voice", "double team", + "dual shot", "dual strike", "enemy", "eye of vengeance", + "fast hands", "fearless", "foresight", + "gunslinger", "hard target", "hatred", + "heightened senses", "hip shooting", "horde fighter", + "hunter of aliens", "independent targeting", + "inspire wrath", "into the jaws of hell", + "iron affinity", "iron discipline", "iron jaw", + "jaded", "killing strike", "leap up", + "leaping dodge", "light sleeper", "lightning attack", + "lightning reflexes", "litany of hate", "logis implant", + "luminen blast", "luminen charge", "luminen shock", + "maglev grace", "marksman", "master craftsman", + "mechadendrite use", "meditation", "mental rage", + "mighty shot", "nerves of steel", "night vision", + "peer", "pistol training", "precise blow", + "psy rating", "quick draw", "rapid reaction", + "rapid reload", "resistance", "rite of fear", + "rite of pure thought", "sacrificial strike", + "sharpshooter", "slayer of daemons", "sound constitution", + "sprint", "step aside", "storm of iron", + "strong minded", "swift attack", "talent", + "thunder charge", "true grit", "two weapon wielder", + "unarmed master", "unarmed warrior", + "unnatural characteristic", "warp affinity", + "weapon training", + # Chapter speciality talents + "astartes assault", "astartes devastator", "astartes librarian", + "astartes tactical", "astartes techmarine", + "basic weapon training", "bolt weapon mastery", + "energy cache", "enhanced bionic frame", + "frenzy", "furious assault", "flesh render", + "grenade mastery", "hammer blow", "hard target", + "head shot", "incendiary charge", "mechadendrite use", + "overcharge", "overwatch", "precision killer", + "stay vigilant", "tactical advance", +} + +def normalise_talent_name(raw): + """Normalise OCR talent name to lowercase clean string.""" + # Remove OCR random-cap artifacts: if string is mixed case with no obvious pattern, + # normalise to lowercase + return re.sub(r'\s+', ' ', raw).strip().lower() + +def split_two_columns(line): + """Split a line from a two-column PDF layout. + Returns (left, right) stripped strings, either may be empty.""" + # Find the largest run of whitespace (gap between columns) + matches = list(re.finditer(r' {5,}', line)) + if not matches: + return line.strip(), "" + # Pick the widest gap; if there are multiple, prefer the one closest to centre + best = max(matches, key=lambda m: m.end() - m.start()) + left = line[:best.start()].strip() + right = line[best.end():].strip() + return left, right + +def looks_like_talent_name(s): + """Heuristic: a talent name is a short phrase (≤8 words) with no digits/punctuation.""" + s = s.strip() + if not s or len(s) > 70 or len(s) < 3: + return False + if re.search(r'[0-9:;|]', s): + return False + words = s.split() + if len(words) > 8: + return False + # Must be mostly letters + letters = sum(c.isalpha() for c in s) + if letters / len(s) < 0.7: + return False + norm = normalise_talent_name(s) + # Must match a known talent or be a plausible name + for kt in KNOWN_TALENTS: + if kt == norm or kt.startswith(norm) or norm.startswith(kt): + return True + # fallback: allow if it looks like a heading (no verb, short) + return False + +def parse_talents_from_cr(cr_path): + """Parse the talent descriptions from CR.txt using column-splitting.""" + with open(cr_path, encoding="utf-8") as f: + lines = f.readlines() + + # Find start of talent descriptions (after the talent table) + START_HINT = 7272 # from analysis + END_HINT = 8450 + + relevant = lines[START_HINT - 1 : END_HINT] + + # Rebuild two separate column streams + left_stream = [] + right_stream = [] + + for line in relevant: + line = line.rstrip('\n') + l, r = split_two_columns(line) + left_stream.append(l) + right_stream.append(r) + + def extract_talents_from_stream(stream): + """Extract {name, prerequisites, description} dicts from one column stream.""" + talents = [] + current = None + prereq_done = False + + for line in stream: + line = line.strip() + if not line: + continue + + # Check if this is a "Prerequisites:" line + pm = re.match(r'^Prerequisites?:\s*(.*)', line, re.IGNORECASE) + if pm and current is not None: + current['prerequisites'] = pm.group(1).strip() + prereq_done = True + continue + + # Check if it looks like a talent name + if looks_like_talent_name(line) and (current is None or prereq_done): + if current and current.get('description'): + talents.append(current) + current = {'name': line, 'prerequisites': '—', 'description': ''} + prereq_done = False + continue + + # Otherwise it's description text + if current is not None: + if current['description']: + current['description'] += ' ' + line + else: + current['description'] = line + + if current and current.get('description'): + talents.append(current) + + return talents + + left_talents = extract_talents_from_stream(left_stream) + right_talents = extract_talents_from_stream(right_stream) + + # Merge and deduplicate by normalised name + all_talents = {} + for t in left_talents + right_talents: + norm = normalise_talent_name(t['name']) + if norm not in all_talents or len(t['description']) > len(all_talents[norm]['description']): + all_talents[norm] = t + + return list(all_talents.values()) + +# ── 3. Direct DB update via mysql.connector ──────────────────────────────────── + +def get_db(): + import mysql.connector + return mysql.connector.connect( + host="192.168.1.113", port=3307, + user="deathwatch", + password=open(REPO / ".env.db").read().strip() if (REPO / ".env.db").exists() else "DwRoller@2025!", + database="deathwatch" + ) + +def run(): + print("=== Deathwatch Rules Updater ===\n") + + # Try direct DB connection + try: + db = get_db() + cursor = db.cursor() + print("Connected to MariaDB directly.\n") + except Exception as e: + print(f"Cannot connect to MariaDB: {e}") + print("Falling back to HTTP API (slower).") + db = None + cursor = None + + def exec_sql(sql, params=()): + if cursor: + cursor.execute(sql, params) + else: + print(f" [no-db] would execute: {sql[:80]}") + + def upsert(rule_id, title, content, page, source, source_abbr, category): + if cursor: + cursor.execute( + """INSERT INTO rules (rule_id, title, content, page, source, source_abbr, category) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + title=VALUES(title), content=VALUES(content), + page=VALUES(page), source=VALUES(source), + source_abbr=VALUES(source_abbr), category=VALUES(category)""", + (rule_id, title, content, page, source, source_abbr, category) + ) + else: + print(f" [no-db] upsert '{title}'") + + # ── Step 1: Delete garbage advance-table entries ────────────────────────── + print("Step 1: Removing garbled advance-table entries...") + garbage_patterns = [ + # Advance tables masquerading as rules + "title LIKE 'AdvanceCostType%'", + # Rules whose entire content is a cost/prerequisites table + "content REGEXP '^[A-Za-z ]+[[:space:]]+[0-9,]+[[:space:]]+(Skill|Talent)[[:space:]]'", + # Very short meaningless entries (under 30 chars content, sanitized source) + "(source = 'sanitized' AND LENGTH(content) < 40)", + ] + deleted = 0 + for pat in garbage_patterns: + if cursor: + cursor.execute(f"SELECT COUNT(*) FROM rules WHERE {pat}") + count = cursor.fetchone()[0] + cursor.execute(f"DELETE FROM rules WHERE {pat}") + print(f" Deleted {count} rows matching: {pat[:60]}") + deleted += count + else: + print(f" [no-db] would delete: {pat[:60]}") + print(f" Total deleted: {deleted}\n") + + # ── Step 2: Skills from CSV ─────────────────────────────────────────────── + print("Step 2: Upserting skills from CSV...") + skills = load_skills() + for skill in skills: + name = skill['name'].strip() + content = build_skill_content(skill) + char = skill.get('characteristic', '').strip() + stype = skill.get('type', '').strip() + desc_header = f"{stype} Skill – {char}" if char else stype + full_content = f"{desc_header}\n\n{content}" if desc_header else content + + rule_id = "skill_" + re.sub(r'[^a-z0-9]', '_', name.lower()) + upsert(rule_id, name, full_content, None, + "Core Rulebook", "CR", "skills") + print(f" ✓ {name}") + if db: + db.commit() + print(f" {len(skills)} skills updated.\n") + + # ── Step 3: Talents from CR.txt ─────────────────────────────────────────── + print("Step 3: Parsing and upserting talents from CR.txt...") + cr_path = REPO / "database" / "rules" / "CR.txt" + talents = parse_talents_from_cr(cr_path) + print(f" Parsed {len(talents)} talent entries from CR.txt") + + imported = 0 + for t in talents: + name = re.sub(r'\s+', ' ', t['name']).strip() + # Clean up OCR capitalisation: convert random-caps to title case + name_clean = re.sub(r'([A-Z])', lambda m: m.group(1).lower(), name) + name_clean = ' '.join(w.capitalize() for w in name_clean.split()) + + prereq = t.get('prerequisites', '—').strip() + desc = t.get('description', '').strip() + if len(desc) < 20: + continue # Skip empty/garbage entries + + content = f"Prerequisites: {prereq}\n\n{desc}" + rule_id = "talent_" + re.sub(r'[^a-z0-9]', '_', name_clean.lower()) + + upsert(rule_id, name_clean, content, None, + "Core Rulebook", "CR", "talents") + imported += 1 + + if db: + db.commit() + print(f" {imported} talents upserted.\n") + + # ── Step 4: Fix existing sanitized rules – remove pure table entries ────── + print("Step 4: Cleaning up remaining garbled sanitized entries...") + if cursor: + # Rules where the title IS the first line of the content (typical for advance tables) + cursor.execute(""" + DELETE FROM rules + WHERE source = 'sanitized' + AND ( + content LIKE '%Cost%Type%Prerequisites%' + OR content REGEXP '[0-9]{3,}[[:space:]]+(Skill|Talent)[[:space:]]' + OR (title = content) + ) + """) + n = cursor.rowcount + db.commit() + print(f" Deleted {n} more garbled entries.\n") + else: + print(" [no-db] skipped.\n") + + # ── Done ────────────────────────────────────────────────────────────────── + if cursor: + cursor.execute("SELECT COUNT(*) FROM rules") + total = cursor.fetchone()[0] + cursor.execute("SELECT source, COUNT(*) FROM rules GROUP BY source ORDER BY COUNT(*) DESC") + by_source = cursor.fetchall() + print(f"=== Done. Total rules in DB: {total} ===") + for src, cnt in by_source: + print(f" {src}: {cnt}") + cursor.close() + db.close() + else: + print("=== Done (dry-run, no DB connection). ===") + +if __name__ == "__main__": + run()