#!/usr/bin/env python3 """ Build a midgame-oriented Deathwatch rules database. This intentionally does not try to blindly turn every PDF heading into a rule. It builds high-value session references from verified local sources, keeps source/page provenance, and marks OCR/text fallbacks explicitly. """ import csv import json import re from datetime import datetime, timezone from pathlib import Path import fitz REPO = Path(__file__).resolve().parent.parent RULES_DIR = REPO / "database" / "rules" DATA_DIR = REPO / "data" BACKUPS_DIR = REPO / "database" / "backups" OUT = RULES_DIR / "rules-database.json" BOOKS = { "CR": {"name": "Core Rulebook", "path": RULES_DIR / "CR.pdf"}, "FF": {"name": "First Founding", "path": RULES_DIR / "FF.pdf"}, "RoB": {"name": "Rites of Battle", "path": RULES_DIR / "RoB.pdf"}, "MoX": {"name": "Mark of the Xenos", "path": RULES_DIR / "MoX.pdf"}, "HtC": {"name": "Honour the Chapter", "path": RULES_DIR / "HtC.pdf"}, "ERR": {"name": "Errata", "path": RULES_DIR / "Errata.pdf"}, } MIDGAME_CATEGORIES = [ "combat", "actions", "damage", "conditions", "hordes", "psychic", "skills", "talents", "traits", "squad_mode", "cohesion", "requisition", "renown", "equipment", "mission", "gm_tables", "errata", "supplements", ] REQUIRED_RULES = [ "Dodge", "Parry", "Full Auto Burst", "Semi-Auto Burst", "Suppressing Fire", "Overwatch", "Pinning", "Righteous Fury", "Critical Damage", "Hordes", "Cohesion Damage", "Maintaining Squad Mode", "Requisition", "Renown", "Fear", ] PLAY_AID_CONTENT = { "Requisition": """Quick Use - The Mission has a Requisition rating set by the GM or mission authority. - Each Battle-Brother gets that many Requisition Points for this Mission. - Total item costs may not exceed the Battle-Brother's available Requisition. - Unspent Requisition does not carry over. - Requisitioned equipment is returned at the end of the Mission. Pooling - The squad may pool Requisition for communal gear or to equip one Battle-Brother with a costly item. - The GM should approve who carries and controls pooled equipment. Included Ammunition and Consumables - Ranged weapons include enough basic ammunition for the Mission unless the GM rules otherwise. - Grenades, missiles, and similar consumables last for the Mission unless tracked separately. - Special Issue Ammunition costs are per single clip. Renown Gate - Paying the Requisition cost is not enough if the item has a Renown requirement. - A Battle-Brother must meet or exceed the item's Renown Rank before he may Requisition it. Availability to Requisition Ubiquitous: 1 Abundant: 2 Plentiful: 3 Common: 4-5 Average: 6-8 Scarce: 9-14 Rare: 15-20 Very Rare: 21-30 Extremely Rare: 31-50 Near Unique: 51-70 Unique: 71+ Mid-play Ruling - If an item is from another 40k RPG line, use Availability to estimate cost. - Add a Renown requirement for rare, sacred, xenos, relic, or Astartes-only gear. - If players challenge the Mission's Requisition rating, let the Kill-team Leader make the argument once; repeated haggling should have social consequences.""", "Renown": """Quick Use - Renown is the Battle-Brother's status within the Deathwatch. - Renown gates access to prestigious, rare, or dangerous armoury items. - An item with a Renown requirement cannot be Requisitioned until the Battle-Brother meets that rank. - When a squad pools Requisition, the intended bearer still needs the required Renown. - Renown increases through service, sacrifice, victory, and Mission rewards. Renown Ranks 0-19: Initiated 20-39: Respected 40-59: Distinguished 60-79: Famed 80+: Hero Mid-play Ruling - Use Renown to say "not yet" even when the squad has enough Requisition. - Let pooled Requisition buy an item only if the intended bearer meets its Renown requirement. - For gear imported from another source, assign a Renown gate when the item is iconic, relic-grade, restricted, or politically sensitive.""", } def clean_text(value): value = str(value or "").replace("\r\n", "\n").replace("\r", "\n") value = value.replace("\u0008", "") value = re.sub(r"[ \t]+", " ", value) value = re.sub(r" *\n *", "\n", value) value = re.sub(r"\n{3,}", "\n\n", value) return value.strip() def slug(value): value = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") return value[:90] or "rule" def pdf_pages(abbr, start, end=None): book = BOOKS[abbr] path = book["path"] if not path.exists(): return "" end = end or start doc = fitz.open(path) chunks = [] for page_num in range(start, min(end, len(doc)) + 1): chunks.append(doc[page_num - 1].get_text("text")) doc.close() return clean_text("\n\n".join(chunks)) def source_method_for(abbr, fallback=False): if fallback: return "text" return "pdf" if BOOKS.get(abbr, {}).get("path", Path()).exists() else "text" def make_rule( title, summary, category, source_abbr, page=None, page_end=None, content="", aliases=None, tags=None, related=None, priority=50, method=None, confidence=0.9, ): book = BOOKS.get(source_abbr, {"name": source_abbr}) rule_id = f"{source_abbr.lower()}-{slug(category)}-{slug(title)}" return { "id": rule_id, "rule_id": rule_id, "title": title, "summary": summary, "content": clean_text(content), "category": category, "tags": sorted(set(tags or [])), "aliases": sorted(set(aliases or [])), "relatedRules": sorted(set(related or [])), "page": page, "pageEnd": page_end or page, "source": book["name"], "sourceAbbr": source_abbr, "sourceMethod": method or source_method_for(source_abbr), "confidence": confidence, "midgamePriority": priority, } def load_skills(): path = REPO / "database" / "deathwatch_skills_p94_107.csv" if not path.exists(): return [] rules = [] with path.open(newline="", encoding="utf-8") as handle: for row in csv.DictReader(handle): name = row["name"].strip() characteristic = row.get("characteristic", "").strip() skill_type = row.get("type", "").strip() descriptors = row.get("descriptors", "").strip() parts = [] if row.get("skill_text"): parts.append(row["skill_text"].strip()) if row.get("skill_description"): parts.append(row["skill_description"].strip()) if row.get("skill_use"): parts.append(f"Skill Use: {row['skill_use'].strip()}") summary_bits = [skill_type, characteristic] if descriptors: summary_bits.append(descriptors) summary = " / ".join([b for b in summary_bits if b]) rules.append( make_rule( title=name, summary=summary, category="skills", source_abbr="CR", page=94, page_end=107, content="\n\n".join(parts), aliases=[f"{name} Skill", characteristic], tags=["skill", skill_type.lower(), descriptors.lower()], related=["Test Difficulty"], priority=75 if name in {"Awareness", "Dodge", "Command", "Medicae", "Intimidate"} else 55, method="csv+pdf", confidence=0.95, ) ) return rules CURATED_SECTIONS = [ # Core adjudication ("Test Difficulty", "Difficulty modifiers and common test bands.", "gm_tables", "CR", 204, 205, ["difficulty", "modifier", "ordinary", "challenging"], ["Skill Tests"], 90), ("Fate Points", "Spend or burn Fate Points during play.", "gm_tables", "CR", 205, 205, ["fate", "reroll", "heal", "initiative"], ["Critical Damage"], 70), ("Movement", "Narrative and structured movement, jumping, swimming, climbing, carrying, throwing, and gravity.", "gm_tables", "CR", 206, 211, ["movement", "jump", "swim", "carry", "throw", "gravity"], ["Actions"], 70), # Cohesion and modes ("Cohesion", "Kill-team Cohesion, Cohesion damage, recovery, and Cohesion Challenges.", "cohesion", "CR", 212, 213, ["cohesion", "cohesion challenge"], ["Squad Mode"], 95), ("Cohesion Damage", "When incoming damage reduces the Kill-team's Cohesion and how the leader can resist it.", "cohesion", "CR", 213, 213, ["cohesion damage", "command test"], ["Cohesion", "Squad Mode"], 100), ("Squad Mode", "Entering, maintaining, leaving, and tracking Squad Mode.", "squad_mode", "CR", 214, 215, ["squad mode", "support range", "changing modes"], ["Cohesion", "Solo Mode"], 95), ("Maintaining Squad Mode", "Conditions that cause a Battle-Brother to drop out of Squad Mode.", "squad_mode", "CR", 215, 215, ["maintain squad mode", "support range"], ["Cohesion Damage"], 100), ("Solo Mode Abilities", "Chapter-linked Solo Mode abilities and improvements.", "squad_mode", "CR", 216, 219, ["solo mode", "chapter ability"], ["Squad Mode"], 70), ("Squad Mode Abilities", "Activating, sustaining, joining, and using Squad Mode abilities.", "squad_mode", "CR", 220, 226, ["bolter assault", "fire support", "regroup", "tactical spacing"], ["Cohesion"], 90), # Mission/session structure ("Missions", "Mission preparation, objectives, complications, rewards, experience, renown, and requisition.", "mission", "CR", 227, 232, ["mission", "objective", "oath", "complication", "reward"], ["Requisition", "Renown"], 75), ("Mission Complications", "Complications that change an operation mid-session.", "mission", "CR", 232, 232, ["complication", "bad beginning", "cut off"], ["Missions"], 80), # Combat ("Combat Overview", "Structured time, initiative, rounds, turns, and surprise.", "combat", "CR", 235, 237, ["combat", "initiative", "surprise", "round", "turn"], ["Actions"], 90), ("Actions", "Full, half, free, reaction, extended, and subtype rules.", "actions", "CR", 237, 238, ["action", "half action", "full action", "reaction"], ["Combat Overview"], 95), ("Combat Actions", "Action descriptions including Aim, Charge, Dodge, Full Auto Burst, Overwatch, Semi-Auto Burst, and Suppressing Fire.", "actions", "CR", 238, 244, ["aim", "charge", "dodge", "full auto", "overwatch", "semi-auto", "suppressing fire", "parry"], ["The Attack"], 100), ("Dodge", "Reaction to negate a hit when allowed.", "actions", "CR", 239, 240, ["dodge reaction"], ["Parry", "The Attack"], 100), ("Parry", "Reaction using Weapon Skill to negate a melee hit.", "actions", "CR", 243, 245, ["parry reaction"], ["Dodge", "The Attack"], 100), ("Full Auto Burst", "Full Action ranged attack with additional hits per degree of success.", "actions", "CR", 240, 240, ["full auto", "automatic fire"], ["Semi-Auto Burst", "Weapon Jams"], 100), ("Semi-Auto Burst", "Ranged attack mode with additional hits every two degrees of success.", "actions", "CR", 242, 243, ["semi auto", "semi-auto"], ["Full Auto Burst", "Weapon Jams"], 100), ("Suppressing Fire", "Full Auto fire to force Pinning and threaten a kill zone.", "actions", "CR", 244, 244, ["suppression", "suppressing fire"], ["Pinning", "Full Auto Burst"], 100), ("Overwatch", "Establish a kill zone and fire when targets enter it.", "actions", "CR", 241, 242, ["overwatch", "kill zone"], ["Suppressing Fire", "Pinning"], 100), ("The Attack", "Attack tests, hit locations, damage rolls, Righteous Fury, and defenses.", "combat", "CR", 245, 246, ["attack", "hit location", "damage roll", "righteous fury"], ["Dodge", "Parry", "Damage"], 100), ("Righteous Fury", "Exploding damage on natural 10s after a successful attack confirmation.", "damage", "CR", 246, 246, ["righteous fury", "damage"], ["The Attack"], 100), ("Combat Circumstances", "Cover, darkness, difficult terrain, melee, range, size, shooting into melee, and prone modifiers.", "combat", "CR", 247, 250, ["cover", "darkness", "range", "size", "prone", "melee"], ["Test Difficulty"], 100), ("Pinning", "Effects of being Pinned and how characters recover.", "conditions", "CR", 249, 249, ["pinned", "pinning test"], ["Suppressing Fire", "Overwatch"], 100), ("Weapon Jams", "Jam thresholds and clearing jammed weapons.", "combat", "CR", 249, 249, ["jam", "weapon jam"], ["Full Auto Burst", "Semi-Auto Burst"], 90), ("Damage", "Wounds, damage types, armour/Toughness reduction, and damage recovery.", "damage", "CR", 251, 251, ["wounds", "damage", "armour", "toughness"], ["Critical Damage"], 100), ("Critical Damage", "Critical damage process and reference tables.", "damage", "CR", 251, 260, ["critical damage", "critical effects", "energy", "explosive", "impact", "rending"], ["Damage"], 100), ("Conditions and Special Damage", "Fatigue, blood loss, fire, falling, suffocation, and special damage states.", "conditions", "CR", 261, 263, ["fatigue", "blood loss", "fire", "falling", "suffocation"], ["Damage"], 95), ("Fear", "Fear tests, degrees of Fear, and mental trauma during play.", "conditions", "CR", 277, 279, ["fear", "shock", "insanity"], ["Test Difficulty"], 100), ("Hordes", "Creating, attacking, damaging, breaking, and running Hordes.", "hordes", "CR", 360, 361, ["horde", "magnitude", "blast weapons", "breaking a horde"], ["Full Auto Burst", "Suppressing Fire"], 100), # Armoury and character-facing rules ("Requisition", "Mission equipment purchasing and Requisition Point ranges.", "requisition", "CR", 139, 140, ["requisition", "req", "availability"], ["Renown", "Missions"], 100), ("Renown", "Renown ranks and their impact on equipment access.", "renown", "CR", 140, 141, ["renown", "requisition", "pooling", "respected", "distinguished", "famed", "hero"], ["Requisition"], 100), ("Craftsmanship", "Poor, common, good, and best craftsmanship effects and costs.", "equipment", "CR", 141, 141, ["craftsmanship", "poor", "good", "best"], ["Requisition"], 80), ("Weapon Profiles", "Weapon class, range, rate of fire, damage, penetration, clip, reload, and special qualities.", "equipment", "CR", 141, 142, ["weapon profile", "rof", "damage", "penetration", "clip", "reload"], ["Weapon Special Qualities"], 90), ("Weapon Special Qualities", "Rules for Accurate, Balanced, Blast, Flame, Tearing, Toxic, Storm, and other qualities.", "equipment", "CR", 143, 145, ["accurate", "balanced", "blast", "flame", "tearing", "toxic", "storm"], ["Weapon Profiles"], 100), ("Astartes Power Armour", "Power armour protection, subsystems, histories, and critical effects.", "equipment", "CR", 160, 164, ["power armour", "armour history", "subsystems"], ["Damage"], 85), ("Tools and Wargear", "Common mission gear, tools, consumables, cybernetics, and relic support rules.", "equipment", "CR", 171, 178, ["wargear", "tools", "drugs", "cybernetics"], ["Requisition"], 75), # Talents, traits, psychic ("Talent Descriptions", "Core talent rules and talent descriptions.", "talents", "CR", 113, 130, ["talent", "prerequisites"], ["Traits"], 80), ("Trait Descriptions", "Creature and character trait rules.", "traits", "CR", 131, 137, ["trait", "unnatural", "fear", "toxic", "warp weapon"], ["Talent Descriptions"], 80), ("Psychic Powers", "Focus Power tests, Psy Rating, fettered/unfettered/push rules, and power use.", "psychic", "CR", 185, 190, ["psychic", "focus power", "psy rating"], ["Psychic Phenomena", "Perils of the Warp"], 90), ("Psychic Phenomena", "Psychic Phenomena results and when they trigger.", "psychic", "CR", 187, 188, ["psychic phenomena", "warp"], ["Perils of the Warp"], 95), ("Perils of the Warp", "Perils table and severe psychic backlash.", "psychic", "CR", 189, 189, ["perils", "warp"], ["Psychic Phenomena"], 95), ] def load_curated_sections(): rules = [] for title, summary, category, abbr, start, end, tags, related, priority in CURATED_SECTIONS: content = PLAY_AID_CONTENT.get(title) or pdf_pages(abbr, start, end) rules.append( make_rule( title=title, summary=summary, category=category, source_abbr=abbr, page=start, page_end=end, content=content, aliases=tags, tags=[category] + tags, related=related, priority=priority, method="pdf+curated" if title in PLAY_AID_CONTENT else None, confidence=0.9 if end - start > 3 else 0.95, ) ) return rules def load_gmk_text_rules(): path = RULES_DIR / "GMK.txt" content = clean_text(path.read_text(encoding="utf-8")) if path.exists() else "" tables = DATA_DIR / "gamemasters_kit" / "gamemasterkit_tables.txt" table_content = clean_text(tables.read_text(encoding="utf-8")) if tables.exists() else "" rules = [] if content: rules.append( make_rule( "Game Master's Kit Reference", "GM Kit mission, antagonist, and table reference text.", "gm_tables", "GMK", page=1, page_end=32, content=content[:50000], aliases=["gm kit", "gamemaster kit", "tables"], tags=["gm", "tables", "reference"], related=["Combat Actions", "Test Difficulty"], priority=70, method="text", confidence=0.75, ) ) if table_content: rules.append( make_rule( "GM Kit Quick Tables", "OCR/text fallback for GM Kit quick-reference tables.", "gm_tables", "GMK", page=None, content=table_content, aliases=["quick tables", "combat tables", "weapon qualities"], tags=["gm", "tables", "ocr"], related=["Combat Actions", "Weapon Special Qualities"], priority=85, method="text+ocr", confidence=0.7, ) ) return rules def load_supplement_rules(): specs = [ ("FF", "First Founding Session Reference", "Chapter rules, additional character options, gear, and chapter-specific material.", ["chapter", "successor", "first founding"]), ("RoB", "Rites of Battle Session Reference", "Expanded squad, mission, vehicle, armoury, and Deathwatch campaign rules.", ["rites of battle", "vehicle", "squad", "mission"]), ("MoX", "Mark of the Xenos Session Reference", "Xenos adversary rules, traits, and GM-facing encounter material.", ["xenos", "adversary", "horde", "trait"]), ("HtC", "Honour the Chapter Session Reference", "Additional Chapter options, deeds, powers, and equipment.", ["honour the chapter", "chapter", "deed"]), ("ERR", "Errata Reference", "Errata and corrections to check when a rule result conflicts.", ["errata", "correction", "faq"]), ] rules = [] for abbr, title, summary, tags in specs: path = BOOKS[abbr]["path"] if not path.exists(): continue doc = fitz.open(path) max_pages = min(len(doc), 8) content = [] for idx in range(max_pages): content.append(doc[idx].get_text("text")) doc.close() rules.append( make_rule( title=title, summary=summary, category="supplements" if abbr != "ERR" else "errata", source_abbr=abbr, page=1, page_end=max_pages, content="\n\n".join(content), aliases=tags, tags=["supplement"] + tags, related=["Requisition", "Talent Descriptions", "Trait Descriptions"], priority=45 if abbr != "ERR" else 90, confidence=0.65, ) ) return rules def dedupe(rules): best = {} for rule in rules: key = rule["rule_id"] if key not in best or rule["midgamePriority"] > best[key]["midgamePriority"]: best[key] = rule return sorted(best.values(), key=lambda r: (-r["midgamePriority"], r["category"], r["title"].lower())) def build_search_index(rules): index = {} for idx, rule in enumerate(rules): fields = [ rule.get("title", ""), rule.get("summary", ""), " ".join(rule.get("tags", [])), " ".join(rule.get("aliases", [])), rule.get("category", ""), ] words = re.findall(r"[a-z0-9]{3,}", " ".join(fields).lower()) for word in set(words): index.setdefault(word, []).append(idx) return index def build_report(rules): by_category = {} by_source = {} missing = [] title_blob = "\n".join((r["title"] + "\n" + " ".join(r.get("aliases", []))).lower() for r in rules) for category in MIDGAME_CATEGORIES: by_category[category] = sum(1 for r in rules if r["category"] == category) for rule in rules: by_source[rule["sourceAbbr"]] = by_source.get(rule["sourceAbbr"], 0) + 1 for required in REQUIRED_RULES: if required.lower() not in title_blob: missing.append(required) low_confidence = [ {"id": r["id"], "title": r["title"], "confidence": r["confidence"], "sourceMethod": r["sourceMethod"]} for r in rules if r["confidence"] < 0.75 ] return { "generatedAt": datetime.now(timezone.utc).isoformat(), "totalRules": len(rules), "byCategory": by_category, "bySource": by_source, "missingRequiredRules": missing, "lowConfidence": low_confidence, } def main(): BACKUPS_DIR.mkdir(parents=True, exist_ok=True) rules = dedupe(load_curated_sections() + load_skills() + load_gmk_text_rules() + load_supplement_rules()) report = build_report(rules) data = { "rules": rules, "searchIndex": build_search_index(rules), "metadata": { "totalRules": len(rules), "sources": [ {"name": info["name"], "abbr": abbr, "available": info["path"].exists()} for abbr, info in BOOKS.items() ], "categories": sorted({r["category"] for r in rules}), "lastUpdated": report["generatedAt"], "builder": "scripts/build-midgame-rules.py", "sourcePolicy": "PDF primary; OCR/text fallback allowed when marked with sourceMethod and confidence.", }, } OUT.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") stamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ") report_path = BACKUPS_DIR / f"rules-midgame-build-report-{stamp}.json" report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") print(f"Wrote {OUT} ({len(rules)} rules)") print(f"Wrote {report_path}") if report["missingRequiredRules"]: print("Missing required rules:", ", ".join(report["missingRequiredRules"])) if __name__ == "__main__": main()