#!/usr/bin/env python3 """ scrape-rules-from-pdfs.py Extracts gear/wargear rule descriptions from Deathwatch PDFs and upserts them into the MariaDB rules table with FULL untruncated text. Uses PyMuPDF to extract bold headings as entry titles and collects all following paragraph text as the rule content — no length limits. Usage: python3 scripts/scrape-rules-from-pdfs.py # all PDFs python3 scripts/scrape-rules-from-pdfs.py CR # only Core Rulebook python3 scripts/scrape-rules-from-pdfs.py CR FF # multiple python3 scripts/scrape-rules-from-pdfs.py --dry-run # print, don't save """ import os, re, sys from pathlib import Path import fitz # PyMuPDF import mysql.connector # ── Config ──────────────────────────────────────────────────────────────────── REPO = Path(__file__).resolve().parent.parent PDF_DIR = REPO / 'database' / 'rules' def load_env_file(path): if not path.exists(): return for line in path.read_text().splitlines(): line = line.strip() if not line or line.startswith('#') or '=' not in line: continue key, value = line.split('=', 1) os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'")) load_env_file(REPO / 'database' / '.env') load_env_file(REPO / '.env') if not os.environ.get('DB_PASSWORD'): raise RuntimeError('DB_PASSWORD must be set in database/.env or .env') DB = dict( host=os.environ.get('DB_HOST', '192.168.1.113'), port=int(os.environ.get('DB_PORT', '3307')), user=os.environ.get('DB_USER', 'deathwatch'), password=os.environ['DB_PASSWORD'], database=os.environ.get('DB_NAME', 'deathwatch'), ) # Book definitions: key → (filename, full_name, abbreviation, page_ranges) # page_ranges = list of (start, end) 1-indexed PDF pages for gear/wargear sections # None = scan entire PDF BOOKS = { 'CR': ('CR.pdf', 'Core Rulebook', 'CR', [(139, 175), (270, 310)]), 'FF': ('FF.pdf', 'First Founding', 'FF', None), 'RoB': ('RoB.pdf', 'Rites of Battle', 'RoB', None), 'MoX': ('MoX.pdf', 'Mark of the Xenos', 'MoX', None), 'HtC': ('HtC.pdf', 'Honour the Chapter','HtC', None), } DRY_RUN = '--dry-run' in sys.argv BOOK_FILTER = [a for a in sys.argv[1:] if not a.startswith('-')] MIN_CONTENT = 50 # minimum chars to keep an entry MAX_CONTENT = 6000 # safety cap # Words that indicate a chapter/section header, not a rule entry CHAPTER_WORDS = { 'chapter', 'table', 'contents', 'index', 'appendix', 'introduction', 'foreword', 'credits', 'special thanks', 'foreword', 'designer', 'developer', 'copyright', } # ── Helpers ─────────────────────────────────────────────────────────────────── def clean(s): s = re.sub(r'[ \t]+', ' ', str(s)) s = re.sub(r' \n', '\n', s) s = re.sub(r'\n ', '\n', s) s = re.sub(r'\n{3,}', '\n\n', s) return s.strip() def normalize_heading(text): """ Fix PDF font artefact where each word is split: 'a staRtes s toRm' → 'Astartes Storm'. Single isolated letter + next token = one word. """ parts = text.split() merged = [] i = 0 while i < len(parts): if len(parts[i]) == 1 and parts[i].isalpha() and i + 1 < len(parts): merged.append(parts[i] + parts[i+1]) i += 2 else: merged.append(parts[i]) i += 1 return ' '.join(w.capitalize() for w in merged) def slug(title): s = re.sub(r'[^a-z0-9]+', '-', title.lower()).strip('-') return s[:80] def looks_like_stat_row(text): """True if line is mostly numbers/dashes (a stat table row).""" tokens = text.split() if len(tokens) < 3: return False num = sum(1 for t in tokens if re.fullmatch(r'[\d\-–—/]+', t)) return num / len(tokens) > 0.55 def is_page_stamp(text): """True if text is just a page number or chapter stamp.""" t = text.strip() return bool(re.fullmatch(r'\d{1,4}', t)) def is_rule_heading(text, is_bold, prev_text=''): """ Heuristic: bold, short (3–80 chars), not a pure number/symbol, not a chapter header, not a stat row. """ if not is_bold: return False t = text.strip() if not (3 <= len(t) <= 80): return False if is_page_stamp(t): return False if looks_like_stat_row(t): return False if re.fullmatch(r'[\d\W]+', t): return False tl = t.lower() if any(tl.startswith(w) for w in CHAPTER_WORDS): return False # Roman numeral chapter headings like "IV : Talents & Traits" if re.match(r'^[IVXivx]+\s*[:\-]', t): return False return True # ── PDF extraction ──────────────────────────────────────────────────────────── def extract_entries(pdf_path, page_ranges=None): """ Walk the PDF's text blocks; detect bold headings as entry titles and accumulate following paragraph text as content. Returns list of {title, content, page} dicts. """ doc = fitz.open(str(pdf_path)) entries = [] cur_title = None cur_lines = [] cur_page = 0 total = len(doc) pages_to_scan = [] if page_ranges: for start, end in page_ranges: pages_to_scan.extend(range(start - 1, min(end, total))) else: pages_to_scan = range(total) def flush(): nonlocal cur_title, cur_lines, cur_page if cur_title: content = clean(' '.join(cur_lines)) if len(content) >= MIN_CONTENT: entries.append({ 'title': cur_title, 'content': content[:MAX_CONTENT], 'page': cur_page, }) cur_title = None cur_lines = [] cur_page = 0 for pg_idx in pages_to_scan: page = doc[pg_idx] pg_no = pg_idx + 1 raw = page.get_text('dict', flags=fitz.TEXT_PRESERVE_WHITESPACE) for block in raw['blocks']: if block['type'] != 0: continue for line in block['lines']: parts, line_bold = [], False for span in line['spans']: txt = span['text'].strip() if not txt: continue if span['flags'] & (1 << 4): line_bold = True parts.append(txt) if not parts: continue text = ' '.join(parts) if is_page_stamp(text): continue if looks_like_stat_row(text): continue if is_rule_heading(text, line_bold): flush() cur_title = normalize_heading(text) cur_page = pg_no elif cur_title: cur_lines.append(text) flush() doc.close() return entries # ── DB upsert ───────────────────────────────────────────────────────────────── def upsert_entries(entries, source, source_abbr): conn = mysql.connector.connect(**DB) cur = conn.cursor() ins = upd = 0 for e in entries: cur.execute(""" INSERT INTO rules (rule_id, title, content, page, source, source_abbr, category) VALUES (%s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE title = VALUES(title), content = VALUES(content), page = VALUES(page), source = VALUES(source), source_abbr= VALUES(source_abbr), category = VALUES(category) """, ( slug(e['title']), e['title'], e['content'], e['page'], source, source_abbr, 'gear' )) if cur.rowcount == 1: ins += 1 elif cur.rowcount == 2: upd += 1 conn.commit() cur.close() conn.close() return ins, upd # ── Main ────────────────────────────────────────────────────────────────────── def main(): targets = BOOK_FILTER if BOOK_FILTER else list(BOOKS.keys()) for abbr in targets: if abbr not in BOOKS: print(f'Unknown book: {abbr} (valid: {list(BOOKS)})') continue fname, source, source_abbr, page_ranges = BOOKS[abbr] pdf_path = PDF_DIR / fname if not pdf_path.exists(): print(f'✗ Not found: {pdf_path}') continue range_str = str(page_ranges) if page_ranges else 'all pages' print(f'\n── {source} [{range_str}]') entries = extract_entries(pdf_path, page_ranges) print(f' Found {len(entries)} entries') if DRY_RUN: for e in entries[:8]: print(f' p{e["page"]:3d} [{e["title"][:45]:<45}] {len(e["content"])} chars') print(f' {e["content"][:100]}…') if len(entries) > 8: print(f' … and {len(entries)-8} more') else: ins, upd = upsert_entries(entries, source, source_abbr) print(f' ✓ Inserted: {ins} Updated: {upd}') print() if __name__ == '__main__': main()