dwroller/scripts/build-midgame-rules.py

#!/usr/bin/env python3
"""
Build a midgame-oriented Deathwatch rules database.

This intentionally does not try to blindly turn every PDF heading into a rule.
It builds high-value session references from verified local sources, keeps
source/page provenance, and marks OCR/text fallbacks explicitly.
"""

import csv
import json
import re
from datetime import datetime, timezone
from pathlib import Path

import fitz

REPO = Path(__file__).resolve().parent.parent
RULES_DIR = REPO / "database" / "rules"
DATA_DIR = REPO / "data"
BACKUPS_DIR = REPO / "database" / "backups"
OUT = RULES_DIR / "rules-database.json"

BOOKS = {
    "CR": {"name": "Core Rulebook", "path": RULES_DIR / "CR.pdf"},
    "FF": {"name": "First Founding", "path": RULES_DIR / "FF.pdf"},
    "RoB": {"name": "Rites of Battle", "path": RULES_DIR / "RoB.pdf"},
    "MoX": {"name": "Mark of the Xenos", "path": RULES_DIR / "MoX.pdf"},
    "HtC": {"name": "Honour the Chapter", "path": RULES_DIR / "HtC.pdf"},
    "ERR": {"name": "Errata", "path": RULES_DIR / "Errata.pdf"},
}

MIDGAME_CATEGORIES = [
    "combat",
    "actions",
    "damage",
    "conditions",
    "hordes",
    "psychic",
    "skills",
    "talents",
    "traits",
    "squad_mode",
    "cohesion",
    "requisition",
    "renown",
    "equipment",
    "mission",
    "gm_tables",
    "errata",
    "supplements",
]

REQUIRED_RULES = [
    "Dodge",
    "Parry",
    "Full Auto Burst",
    "Semi-Auto Burst",
    "Suppressing Fire",
    "Overwatch",
    "Pinning",
    "Righteous Fury",
    "Critical Damage",
    "Hordes",
    "Cohesion Damage",
    "Maintaining Squad Mode",
    "Requisition",
    "Renown",
    "Fear",
]

PLAY_AID_CONTENT = {
    "Requisition": """Quick Use
- The Mission has a Requisition rating set by the GM or mission authority.
- Each Battle-Brother gets that many Requisition Points for this Mission.
- Total item costs may not exceed the Battle-Brother's available Requisition.
- Unspent Requisition does not carry over.
- Requisitioned equipment is returned at the end of the Mission.

Pooling
- The squad may pool Requisition for communal gear or to equip one Battle-Brother with a costly item.
- The GM should approve who carries and controls pooled equipment.

Included Ammunition and Consumables
- Ranged weapons include enough basic ammunition for the Mission unless the GM rules otherwise.
- Grenades, missiles, and similar consumables last for the Mission unless tracked separately.
- Special Issue Ammunition costs are per single clip.

Renown Gate
- Paying the Requisition cost is not enough if the item has a Renown requirement.
- A Battle-Brother must meet or exceed the item's Renown Rank before he may Requisition it.

Availability to Requisition
Ubiquitous: 1
Abundant: 2
Plentiful: 3
Common: 4-5
Average: 6-8
Scarce: 9-14
Rare: 15-20
Very Rare: 21-30
Extremely Rare: 31-50
Near Unique: 51-70
Unique: 71+

Mid-play Ruling
- If an item is from another 40k RPG line, use Availability to estimate cost.
- Add a Renown requirement for rare, sacred, xenos, relic, or Astartes-only gear.
- If players challenge the Mission's Requisition rating, let the Kill-team Leader make the argument once; repeated haggling should have social consequences.""",
    "Renown": """Quick Use
- Renown is the Battle-Brother's status within the Deathwatch.
- Renown gates access to prestigious, rare, or dangerous armoury items.
- An item with a Renown requirement cannot be Requisitioned until the Battle-Brother meets that rank.
- When a squad pools Requisition, the intended bearer still needs the required Renown.
- Renown increases through service, sacrifice, victory, and Mission rewards.

Renown Ranks
0-19: Initiated
20-39: Respected
40-59: Distinguished
60-79: Famed
80+: Hero

Mid-play Ruling
- Use Renown to say "not yet" even when the squad has enough Requisition.
- Let pooled Requisition buy an item only if the intended bearer meets its Renown requirement.
- For gear imported from another source, assign a Renown gate when the item is iconic, relic-grade, restricted, or politically sensitive.""",
}


def clean_text(value):
    value = str(value or "").replace("\r\n", "\n").replace("\r", "\n")
    value = value.replace("\u0008", "")
    value = re.sub(r"[ \t]+", " ", value)
    value = re.sub(r" *\n *", "\n", value)
    value = re.sub(r"\n{3,}", "\n\n", value)
    return value.strip()


def slug(value):
    value = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
    return value[:90] or "rule"


def pdf_pages(abbr, start, end=None):
    book = BOOKS[abbr]
    path = book["path"]
    if not path.exists():
        return ""
    end = end or start
    doc = fitz.open(path)
    chunks = []
    for page_num in range(start, min(end, len(doc)) + 1):
        chunks.append(doc[page_num - 1].get_text("text"))
    doc.close()
    return clean_text("\n\n".join(chunks))


def source_method_for(abbr, fallback=False):
    if fallback:
        return "text"
    return "pdf" if BOOKS.get(abbr, {}).get("path", Path()).exists() else "text"


def make_rule(
    title,
    summary,
    category,
    source_abbr,
    page=None,
    page_end=None,
    content="",
    aliases=None,
    tags=None,
    related=None,
    priority=50,
    method=None,
    confidence=0.9,
):
    book = BOOKS.get(source_abbr, {"name": source_abbr})
    rule_id = f"{source_abbr.lower()}-{slug(category)}-{slug(title)}"
    return {
        "id": rule_id,
        "rule_id": rule_id,
        "title": title,
        "summary": summary,
        "content": clean_text(content),
        "category": category,
        "tags": sorted(set(tags or [])),
        "aliases": sorted(set(aliases or [])),
        "relatedRules": sorted(set(related or [])),
        "page": page,
        "pageEnd": page_end or page,
        "source": book["name"],
        "sourceAbbr": source_abbr,
        "sourceMethod": method or source_method_for(source_abbr),
        "confidence": confidence,
        "midgamePriority": priority,
    }


def load_skills():
    path = REPO / "database" / "deathwatch_skills_p94_107.csv"
    if not path.exists():
        return []
    rules = []
    with path.open(newline="", encoding="utf-8") as handle:
        for row in csv.DictReader(handle):
            name = row["name"].strip()
            characteristic = row.get("characteristic", "").strip()
            skill_type = row.get("type", "").strip()
            descriptors = row.get("descriptors", "").strip()
            parts = []
            if row.get("skill_text"):
                parts.append(row["skill_text"].strip())
            if row.get("skill_description"):
                parts.append(row["skill_description"].strip())
            if row.get("skill_use"):
                parts.append(f"Skill Use: {row['skill_use'].strip()}")
            summary_bits = [skill_type, characteristic]
            if descriptors:
                summary_bits.append(descriptors)
            summary = " / ".join([b for b in summary_bits if b])
            rules.append(
                make_rule(
                    title=name,
                    summary=summary,
                    category="skills",
                    source_abbr="CR",
                    page=94,
                    page_end=107,
                    content="\n\n".join(parts),
                    aliases=[f"{name} Skill", characteristic],
                    tags=["skill", skill_type.lower(), descriptors.lower()],
                    related=["Test Difficulty"],
                    priority=75 if name in {"Awareness", "Dodge", "Command", "Medicae", "Intimidate"} else 55,
                    method="csv+pdf",
                    confidence=0.95,
                )
            )
    return rules


CURATED_SECTIONS = [
    # Core adjudication
    ("Test Difficulty", "Difficulty modifiers and common test bands.", "gm_tables", "CR", 204, 205, ["difficulty", "modifier", "ordinary", "challenging"], ["Skill Tests"], 90),
    ("Fate Points", "Spend or burn Fate Points during play.", "gm_tables", "CR", 205, 205, ["fate", "reroll", "heal", "initiative"], ["Critical Damage"], 70),
    ("Movement", "Narrative and structured movement, jumping, swimming, climbing, carrying, throwing, and gravity.", "gm_tables", "CR", 206, 211, ["movement", "jump", "swim", "carry", "throw", "gravity"], ["Actions"], 70),
    # Cohesion and modes
    ("Cohesion", "Kill-team Cohesion, Cohesion damage, recovery, and Cohesion Challenges.", "cohesion", "CR", 212, 213, ["cohesion", "cohesion challenge"], ["Squad Mode"], 95),
    ("Cohesion Damage", "When incoming damage reduces the Kill-team's Cohesion and how the leader can resist it.", "cohesion", "CR", 213, 213, ["cohesion damage", "command test"], ["Cohesion", "Squad Mode"], 100),
    ("Squad Mode", "Entering, maintaining, leaving, and tracking Squad Mode.", "squad_mode", "CR", 214, 215, ["squad mode", "support range", "changing modes"], ["Cohesion", "Solo Mode"], 95),
    ("Maintaining Squad Mode", "Conditions that cause a Battle-Brother to drop out of Squad Mode.", "squad_mode", "CR", 215, 215, ["maintain squad mode", "support range"], ["Cohesion Damage"], 100),
    ("Solo Mode Abilities", "Chapter-linked Solo Mode abilities and improvements.", "squad_mode", "CR", 216, 219, ["solo mode", "chapter ability"], ["Squad Mode"], 70),
    ("Squad Mode Abilities", "Activating, sustaining, joining, and using Squad Mode abilities.", "squad_mode", "CR", 220, 226, ["bolter assault", "fire support", "regroup", "tactical spacing"], ["Cohesion"], 90),
    # Mission/session structure
    ("Missions", "Mission preparation, objectives, complications, rewards, experience, renown, and requisition.", "mission", "CR", 227, 232, ["mission", "objective", "oath", "complication", "reward"], ["Requisition", "Renown"], 75),
    ("Mission Complications", "Complications that change an operation mid-session.", "mission", "CR", 232, 232, ["complication", "bad beginning", "cut off"], ["Missions"], 80),
    # Combat
    ("Combat Overview", "Structured time, initiative, rounds, turns, and surprise.", "combat", "CR", 235, 237, ["combat", "initiative", "surprise", "round", "turn"], ["Actions"], 90),
    ("Actions", "Full, half, free, reaction, extended, and subtype rules.", "actions", "CR", 237, 238, ["action", "half action", "full action", "reaction"], ["Combat Overview"], 95),
    ("Combat Actions", "Action descriptions including Aim, Charge, Dodge, Full Auto Burst, Overwatch, Semi-Auto Burst, and Suppressing Fire.", "actions", "CR", 238, 244, ["aim", "charge", "dodge", "full auto", "overwatch", "semi-auto", "suppressing fire", "parry"], ["The Attack"], 100),
    ("Dodge", "Reaction to negate a hit when allowed.", "actions", "CR", 239, 240, ["dodge reaction"], ["Parry", "The Attack"], 100),
    ("Parry", "Reaction using Weapon Skill to negate a melee hit.", "actions", "CR", 243, 245, ["parry reaction"], ["Dodge", "The Attack"], 100),
    ("Full Auto Burst", "Full Action ranged attack with additional hits per degree of success.", "actions", "CR", 240, 240, ["full auto", "automatic fire"], ["Semi-Auto Burst", "Weapon Jams"], 100),
    ("Semi-Auto Burst", "Ranged attack mode with additional hits every two degrees of success.", "actions", "CR", 242, 243, ["semi auto", "semi-auto"], ["Full Auto Burst", "Weapon Jams"], 100),
    ("Suppressing Fire", "Full Auto fire to force Pinning and threaten a kill zone.", "actions", "CR", 244, 244, ["suppression", "suppressing fire"], ["Pinning", "Full Auto Burst"], 100),
    ("Overwatch", "Establish a kill zone and fire when targets enter it.", "actions", "CR", 241, 242, ["overwatch", "kill zone"], ["Suppressing Fire", "Pinning"], 100),
    ("The Attack", "Attack tests, hit locations, damage rolls, Righteous Fury, and defenses.", "combat", "CR", 245, 246, ["attack", "hit location", "damage roll", "righteous fury"], ["Dodge", "Parry", "Damage"], 100),
    ("Righteous Fury", "Exploding damage on natural 10s after a successful attack confirmation.", "damage", "CR", 246, 246, ["righteous fury", "damage"], ["The Attack"], 100),
    ("Combat Circumstances", "Cover, darkness, difficult terrain, melee, range, size, shooting into melee, and prone modifiers.", "combat", "CR", 247, 250, ["cover", "darkness", "range", "size", "prone", "melee"], ["Test Difficulty"], 100),
    ("Pinning", "Effects of being Pinned and how characters recover.", "conditions", "CR", 249, 249, ["pinned", "pinning test"], ["Suppressing Fire", "Overwatch"], 100),
    ("Weapon Jams", "Jam thresholds and clearing jammed weapons.", "combat", "CR", 249, 249, ["jam", "weapon jam"], ["Full Auto Burst", "Semi-Auto Burst"], 90),
    ("Damage", "Wounds, damage types, armour/Toughness reduction, and damage recovery.", "damage", "CR", 251, 251, ["wounds", "damage", "armour", "toughness"], ["Critical Damage"], 100),
    ("Critical Damage", "Critical damage process and reference tables.", "damage", "CR", 251, 260, ["critical damage", "critical effects", "energy", "explosive", "impact", "rending"], ["Damage"], 100),
    ("Conditions and Special Damage", "Fatigue, blood loss, fire, falling, suffocation, and special damage states.", "conditions", "CR", 261, 263, ["fatigue", "blood loss", "fire", "falling", "suffocation"], ["Damage"], 95),
    ("Fear", "Fear tests, degrees of Fear, and mental trauma during play.", "conditions", "CR", 277, 279, ["fear", "shock", "insanity"], ["Test Difficulty"], 100),
    ("Hordes", "Creating, attacking, damaging, breaking, and running Hordes.", "hordes", "CR", 360, 361, ["horde", "magnitude", "blast weapons", "breaking a horde"], ["Full Auto Burst", "Suppressing Fire"], 100),
    # Armoury and character-facing rules
    ("Requisition", "Mission equipment purchasing and Requisition Point ranges.", "requisition", "CR", 139, 140, ["requisition", "req", "availability"], ["Renown", "Missions"], 100),
    ("Renown", "Renown ranks and their impact on equipment access.", "renown", "CR", 140, 141, ["renown", "requisition", "pooling", "respected", "distinguished", "famed", "hero"], ["Requisition"], 100),
    ("Craftsmanship", "Poor, common, good, and best craftsmanship effects and costs.", "equipment", "CR", 141, 141, ["craftsmanship", "poor", "good", "best"], ["Requisition"], 80),
    ("Weapon Profiles", "Weapon class, range, rate of fire, damage, penetration, clip, reload, and special qualities.", "equipment", "CR", 141, 142, ["weapon profile", "rof", "damage", "penetration", "clip", "reload"], ["Weapon Special Qualities"], 90),
    ("Weapon Special Qualities", "Rules for Accurate, Balanced, Blast, Flame, Tearing, Toxic, Storm, and other qualities.", "equipment", "CR", 143, 145, ["accurate", "balanced", "blast", "flame", "tearing", "toxic", "storm"], ["Weapon Profiles"], 100),
    ("Astartes Power Armour", "Power armour protection, subsystems, histories, and critical effects.", "equipment", "CR", 160, 164, ["power armour", "armour history", "subsystems"], ["Damage"], 85),
    ("Tools and Wargear", "Common mission gear, tools, consumables, cybernetics, and relic support rules.", "equipment", "CR", 171, 178, ["wargear", "tools", "drugs", "cybernetics"], ["Requisition"], 75),
    # Talents, traits, psychic
    ("Talent Descriptions", "Core talent rules and talent descriptions.", "talents", "CR", 113, 130, ["talent", "prerequisites"], ["Traits"], 80),
    ("Trait Descriptions", "Creature and character trait rules.", "traits", "CR", 131, 137, ["trait", "unnatural", "fear", "toxic", "warp weapon"], ["Talent Descriptions"], 80),
    ("Psychic Powers", "Focus Power tests, Psy Rating, fettered/unfettered/push rules, and power use.", "psychic", "CR", 185, 190, ["psychic", "focus power", "psy rating"], ["Psychic Phenomena", "Perils of the Warp"], 90),
    ("Psychic Phenomena", "Psychic Phenomena results and when they trigger.", "psychic", "CR", 187, 188, ["psychic phenomena", "warp"], ["Perils of the Warp"], 95),
    ("Perils of the Warp", "Perils table and severe psychic backlash.", "psychic", "CR", 189, 189, ["perils", "warp"], ["Psychic Phenomena"], 95),
]


def load_curated_sections():
    rules = []
    for title, summary, category, abbr, start, end, tags, related, priority in CURATED_SECTIONS:
        content = PLAY_AID_CONTENT.get(title) or pdf_pages(abbr, start, end)
        rules.append(
            make_rule(
                title=title,
                summary=summary,
                category=category,
                source_abbr=abbr,
                page=start,
                page_end=end,
                content=content,
                aliases=tags,
                tags=[category] + tags,
                related=related,
                priority=priority,
                method="pdf+curated" if title in PLAY_AID_CONTENT else None,
                confidence=0.9 if end - start > 3 else 0.95,
            )
        )
    return rules


def load_gmk_text_rules():
    path = RULES_DIR / "GMK.txt"
    content = clean_text(path.read_text(encoding="utf-8")) if path.exists() else ""
    tables = DATA_DIR / "gamemasters_kit" / "gamemasterkit_tables.txt"
    table_content = clean_text(tables.read_text(encoding="utf-8")) if tables.exists() else ""
    rules = []
    if content:
        rules.append(
            make_rule(
                "Game Master's Kit Reference",
                "GM Kit mission, antagonist, and table reference text.",
                "gm_tables",
                "GMK",
                page=1,
                page_end=32,
                content=content[:50000],
                aliases=["gm kit", "gamemaster kit", "tables"],
                tags=["gm", "tables", "reference"],
                related=["Combat Actions", "Test Difficulty"],
                priority=70,
                method="text",
                confidence=0.75,
            )
        )
    if table_content:
        rules.append(
            make_rule(
                "GM Kit Quick Tables",
                "OCR/text fallback for GM Kit quick-reference tables.",
                "gm_tables",
                "GMK",
                page=None,
                content=table_content,
                aliases=["quick tables", "combat tables", "weapon qualities"],
                tags=["gm", "tables", "ocr"],
                related=["Combat Actions", "Weapon Special Qualities"],
                priority=85,
                method="text+ocr",
                confidence=0.7,
            )
        )
    return rules


def load_supplement_rules():
    specs = [
        ("FF", "First Founding Session Reference", "Chapter rules, additional character options, gear, and chapter-specific material.", ["chapter", "successor", "first founding"]),
        ("RoB", "Rites of Battle Session Reference", "Expanded squad, mission, vehicle, armoury, and Deathwatch campaign rules.", ["rites of battle", "vehicle", "squad", "mission"]),
        ("MoX", "Mark of the Xenos Session Reference", "Xenos adversary rules, traits, and GM-facing encounter material.", ["xenos", "adversary", "horde", "trait"]),
        ("HtC", "Honour the Chapter Session Reference", "Additional Chapter options, deeds, powers, and equipment.", ["honour the chapter", "chapter", "deed"]),
        ("ERR", "Errata Reference", "Errata and corrections to check when a rule result conflicts.", ["errata", "correction", "faq"]),
    ]
    rules = []
    for abbr, title, summary, tags in specs:
        path = BOOKS[abbr]["path"]
        if not path.exists():
            continue
        doc = fitz.open(path)
        max_pages = min(len(doc), 8)
        content = []
        for idx in range(max_pages):
            content.append(doc[idx].get_text("text"))
        doc.close()
        rules.append(
            make_rule(
                title=title,
                summary=summary,
                category="supplements" if abbr != "ERR" else "errata",
                source_abbr=abbr,
                page=1,
                page_end=max_pages,
                content="\n\n".join(content),
                aliases=tags,
                tags=["supplement"] + tags,
                related=["Requisition", "Talent Descriptions", "Trait Descriptions"],
                priority=45 if abbr != "ERR" else 90,
                confidence=0.65,
            )
        )
    return rules


def dedupe(rules):
    best = {}
    for rule in rules:
        key = rule["rule_id"]
        if key not in best or rule["midgamePriority"] > best[key]["midgamePriority"]:
            best[key] = rule
    return sorted(best.values(), key=lambda r: (-r["midgamePriority"], r["category"], r["title"].lower()))


def build_search_index(rules):
    index = {}
    for idx, rule in enumerate(rules):
        fields = [
            rule.get("title", ""),
            rule.get("summary", ""),
            " ".join(rule.get("tags", [])),
            " ".join(rule.get("aliases", [])),
            rule.get("category", ""),
        ]
        words = re.findall(r"[a-z0-9]{3,}", " ".join(fields).lower())
        for word in set(words):
            index.setdefault(word, []).append(idx)
    return index


def build_report(rules):
    by_category = {}
    by_source = {}
    missing = []
    title_blob = "\n".join((r["title"] + "\n" + " ".join(r.get("aliases", []))).lower() for r in rules)
    for category in MIDGAME_CATEGORIES:
        by_category[category] = sum(1 for r in rules if r["category"] == category)
    for rule in rules:
        by_source[rule["sourceAbbr"]] = by_source.get(rule["sourceAbbr"], 0) + 1
    for required in REQUIRED_RULES:
        if required.lower() not in title_blob:
            missing.append(required)
    low_confidence = [
        {"id": r["id"], "title": r["title"], "confidence": r["confidence"], "sourceMethod": r["sourceMethod"]}
        for r in rules
        if r["confidence"] < 0.75
    ]
    return {
        "generatedAt": datetime.now(timezone.utc).isoformat(),
        "totalRules": len(rules),
        "byCategory": by_category,
        "bySource": by_source,
        "missingRequiredRules": missing,
        "lowConfidence": low_confidence,
    }


def main():
    BACKUPS_DIR.mkdir(parents=True, exist_ok=True)
    rules = dedupe(load_curated_sections() + load_skills() + load_gmk_text_rules() + load_supplement_rules())
    report = build_report(rules)
    data = {
        "rules": rules,
        "searchIndex": build_search_index(rules),
        "metadata": {
            "totalRules": len(rules),
            "sources": [
                {"name": info["name"], "abbr": abbr, "available": info["path"].exists()}
                for abbr, info in BOOKS.items()
            ],
            "categories": sorted({r["category"] for r in rules}),
            "lastUpdated": report["generatedAt"],
            "builder": "scripts/build-midgame-rules.py",
            "sourcePolicy": "PDF primary; OCR/text fallback allowed when marked with sourceMethod and confidence.",
        },
    }
    OUT.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
    stamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ")
    report_path = BACKUPS_DIR / f"rules-midgame-build-report-{stamp}.json"
    report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
    print(f"Wrote {OUT} ({len(rules)} rules)")
    print(f"Wrote {report_path}")
    if report["missingRequiredRules"]:
        print("Missing required rules:", ", ".join(report["missingRequiredRules"]))


if __name__ == "__main__":
    main()