Add weapon images, full rule scraping, and expanded quick reference
- scripts/generate-weapon-images.js: AI image generation for weapon cards supporting HuggingFace FLUX.1-schnell, OpenAI DALL-E 3, and Gemini Imagen; images served from public/weapon-images/ via new static route in server.js - public/cards.html: load manifest.json and show AI art on weapon cards - scripts/scrape-rules-from-pdfs.py: PyMuPDF scraper extracting full rule text from all five rulebook PDFs with bold-span heading detection - scripts/tag-and-dedup-rules.js: standalone dedup helper (JS-side grouping) - database/routes/rulesRoutes.js: remove content truncation in search/random, add admin dedup endpoint (JS-side, fast), fix RulesTab to fetch full content - src/components/RulesTab.jsx: fetch full rule content on modal open - public/print.html: expanded quick reference with pre-gen character stat blocks (Sepheran + Lucian), Astartes traits sheet with correct derived SBs and TBs, full Critical Hit Tables for all 4 locations × 4 damage types Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
258
scripts/scrape-rules-from-pdfs.py
Normal file
258
scripts/scrape-rules-from-pdfs.py
Normal file
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
scrape-rules-from-pdfs.py
|
||||
|
||||
Extracts gear/wargear rule descriptions from Deathwatch PDFs and upserts
|
||||
them into the MariaDB rules table with FULL untruncated text.
|
||||
|
||||
Uses PyMuPDF to extract bold headings as entry titles and collects all
|
||||
following paragraph text as the rule content — no length limits.
|
||||
|
||||
Usage:
|
||||
python3 scripts/scrape-rules-from-pdfs.py # all PDFs
|
||||
python3 scripts/scrape-rules-from-pdfs.py CR # only Core Rulebook
|
||||
python3 scripts/scrape-rules-from-pdfs.py CR FF # multiple
|
||||
python3 scripts/scrape-rules-from-pdfs.py --dry-run # print, don't save
|
||||
"""
|
||||
|
||||
import re, sys
|
||||
from pathlib import Path
|
||||
import fitz # PyMuPDF
|
||||
import mysql.connector
|
||||
|
||||
# ── Config ────────────────────────────────────────────────────────────────────
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
PDF_DIR = REPO / 'database' / 'rules'
|
||||
|
||||
DB = dict(
|
||||
host='192.168.1.113', port=3307, user='deathwatch',
|
||||
password='DwRoller@2025!', database='deathwatch'
|
||||
)
|
||||
|
||||
# Book definitions: key → (filename, full_name, abbreviation, page_ranges)
|
||||
# page_ranges = list of (start, end) 1-indexed PDF pages for gear/wargear sections
|
||||
# None = scan entire PDF
|
||||
BOOKS = {
|
||||
'CR': ('CR.pdf', 'Core Rulebook', 'CR', [(139, 175), (270, 310)]),
|
||||
'FF': ('FF.pdf', 'First Founding', 'FF', None),
|
||||
'RoB': ('RoB.pdf', 'Rites of Battle', 'RoB', None),
|
||||
'MoX': ('MoX.pdf', 'Mark of the Xenos', 'MoX', None),
|
||||
'HtC': ('HtC.pdf', 'Honour the Chapter','HtC', None),
|
||||
}
|
||||
|
||||
DRY_RUN = '--dry-run' in sys.argv
|
||||
BOOK_FILTER = [a for a in sys.argv[1:] if not a.startswith('-')]
|
||||
|
||||
MIN_CONTENT = 50 # minimum chars to keep an entry
|
||||
MAX_CONTENT = 6000 # safety cap
|
||||
|
||||
# Words that indicate a chapter/section header, not a rule entry
|
||||
CHAPTER_WORDS = {
|
||||
'chapter', 'table', 'contents', 'index', 'appendix', 'introduction',
|
||||
'foreword', 'credits', 'special thanks', 'foreword', 'designer',
|
||||
'developer', 'copyright',
|
||||
}
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
def clean(s):
|
||||
s = re.sub(r'[ \t]+', ' ', str(s))
|
||||
s = re.sub(r' \n', '\n', s)
|
||||
s = re.sub(r'\n ', '\n', s)
|
||||
s = re.sub(r'\n{3,}', '\n\n', s)
|
||||
return s.strip()
|
||||
|
||||
def normalize_heading(text):
|
||||
"""
|
||||
Fix PDF font artefact where each word is split: 'a staRtes s toRm' → 'Astartes Storm'.
|
||||
Single isolated letter + next token = one word.
|
||||
"""
|
||||
parts = text.split()
|
||||
merged = []
|
||||
i = 0
|
||||
while i < len(parts):
|
||||
if len(parts[i]) == 1 and parts[i].isalpha() and i + 1 < len(parts):
|
||||
merged.append(parts[i] + parts[i+1])
|
||||
i += 2
|
||||
else:
|
||||
merged.append(parts[i])
|
||||
i += 1
|
||||
return ' '.join(w.capitalize() for w in merged)
|
||||
|
||||
def slug(title):
|
||||
s = re.sub(r'[^a-z0-9]+', '-', title.lower()).strip('-')
|
||||
return s[:80]
|
||||
|
||||
def looks_like_stat_row(text):
|
||||
"""True if line is mostly numbers/dashes (a stat table row)."""
|
||||
tokens = text.split()
|
||||
if len(tokens) < 3:
|
||||
return False
|
||||
num = sum(1 for t in tokens if re.fullmatch(r'[\d\-–—/]+', t))
|
||||
return num / len(tokens) > 0.55
|
||||
|
||||
def is_page_stamp(text):
|
||||
"""True if text is just a page number or chapter stamp."""
|
||||
t = text.strip()
|
||||
return bool(re.fullmatch(r'\d{1,4}', t))
|
||||
|
||||
def is_rule_heading(text, is_bold, prev_text=''):
|
||||
"""
|
||||
Heuristic: bold, short (3–80 chars), not a pure number/symbol,
|
||||
not a chapter header, not a stat row.
|
||||
"""
|
||||
if not is_bold:
|
||||
return False
|
||||
t = text.strip()
|
||||
if not (3 <= len(t) <= 80):
|
||||
return False
|
||||
if is_page_stamp(t):
|
||||
return False
|
||||
if looks_like_stat_row(t):
|
||||
return False
|
||||
if re.fullmatch(r'[\d\W]+', t):
|
||||
return False
|
||||
tl = t.lower()
|
||||
if any(tl.startswith(w) for w in CHAPTER_WORDS):
|
||||
return False
|
||||
# Roman numeral chapter headings like "IV : Talents & Traits"
|
||||
if re.match(r'^[IVXivx]+\s*[:\-]', t):
|
||||
return False
|
||||
return True
|
||||
|
||||
# ── PDF extraction ────────────────────────────────────────────────────────────
|
||||
def extract_entries(pdf_path, page_ranges=None):
|
||||
"""
|
||||
Walk the PDF's text blocks; detect bold headings as entry titles and
|
||||
accumulate following paragraph text as content.
|
||||
Returns list of {title, content, page} dicts.
|
||||
"""
|
||||
doc = fitz.open(str(pdf_path))
|
||||
entries = []
|
||||
cur_title = None
|
||||
cur_lines = []
|
||||
cur_page = 0
|
||||
|
||||
total = len(doc)
|
||||
pages_to_scan = []
|
||||
if page_ranges:
|
||||
for start, end in page_ranges:
|
||||
pages_to_scan.extend(range(start - 1, min(end, total)))
|
||||
else:
|
||||
pages_to_scan = range(total)
|
||||
|
||||
def flush():
|
||||
nonlocal cur_title, cur_lines, cur_page
|
||||
if cur_title:
|
||||
content = clean(' '.join(cur_lines))
|
||||
if len(content) >= MIN_CONTENT:
|
||||
entries.append({
|
||||
'title': cur_title,
|
||||
'content': content[:MAX_CONTENT],
|
||||
'page': cur_page,
|
||||
})
|
||||
cur_title = None
|
||||
cur_lines = []
|
||||
cur_page = 0
|
||||
|
||||
for pg_idx in pages_to_scan:
|
||||
page = doc[pg_idx]
|
||||
pg_no = pg_idx + 1
|
||||
raw = page.get_text('dict', flags=fitz.TEXT_PRESERVE_WHITESPACE)
|
||||
|
||||
for block in raw['blocks']:
|
||||
if block['type'] != 0:
|
||||
continue
|
||||
for line in block['lines']:
|
||||
parts, line_bold = [], False
|
||||
for span in line['spans']:
|
||||
txt = span['text'].strip()
|
||||
if not txt:
|
||||
continue
|
||||
if span['flags'] & (1 << 4):
|
||||
line_bold = True
|
||||
parts.append(txt)
|
||||
|
||||
if not parts:
|
||||
continue
|
||||
text = ' '.join(parts)
|
||||
|
||||
if is_page_stamp(text):
|
||||
continue
|
||||
if looks_like_stat_row(text):
|
||||
continue
|
||||
|
||||
if is_rule_heading(text, line_bold):
|
||||
flush()
|
||||
cur_title = normalize_heading(text)
|
||||
cur_page = pg_no
|
||||
elif cur_title:
|
||||
cur_lines.append(text)
|
||||
|
||||
flush()
|
||||
doc.close()
|
||||
return entries
|
||||
|
||||
# ── DB upsert ─────────────────────────────────────────────────────────────────
|
||||
def upsert_entries(entries, source, source_abbr):
|
||||
conn = mysql.connector.connect(**DB)
|
||||
cur = conn.cursor()
|
||||
ins = upd = 0
|
||||
for e in entries:
|
||||
cur.execute("""
|
||||
INSERT INTO rules (rule_id, title, content, page, source, source_abbr, category)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
title = VALUES(title),
|
||||
content = VALUES(content),
|
||||
page = VALUES(page),
|
||||
source = VALUES(source),
|
||||
source_abbr= VALUES(source_abbr),
|
||||
category = VALUES(category)
|
||||
""", (
|
||||
slug(e['title']), e['title'], e['content'],
|
||||
e['page'], source, source_abbr, 'gear'
|
||||
))
|
||||
if cur.rowcount == 1:
|
||||
ins += 1
|
||||
elif cur.rowcount == 2:
|
||||
upd += 1
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
return ins, upd
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
def main():
|
||||
targets = BOOK_FILTER if BOOK_FILTER else list(BOOKS.keys())
|
||||
|
||||
for abbr in targets:
|
||||
if abbr not in BOOKS:
|
||||
print(f'Unknown book: {abbr} (valid: {list(BOOKS)})')
|
||||
continue
|
||||
|
||||
fname, source, source_abbr, page_ranges = BOOKS[abbr]
|
||||
pdf_path = PDF_DIR / fname
|
||||
if not pdf_path.exists():
|
||||
print(f'✗ Not found: {pdf_path}')
|
||||
continue
|
||||
|
||||
range_str = str(page_ranges) if page_ranges else 'all pages'
|
||||
print(f'\n── {source} [{range_str}]')
|
||||
|
||||
entries = extract_entries(pdf_path, page_ranges)
|
||||
print(f' Found {len(entries)} entries')
|
||||
|
||||
if DRY_RUN:
|
||||
for e in entries[:8]:
|
||||
print(f' p{e["page"]:3d} [{e["title"][:45]:<45}] {len(e["content"])} chars')
|
||||
print(f' {e["content"][:100]}…')
|
||||
if len(entries) > 8:
|
||||
print(f' … and {len(entries)-8} more')
|
||||
else:
|
||||
ins, upd = upsert_entries(entries, source, source_abbr)
|
||||
print(f' ✓ Inserted: {ins} Updated: {upd}')
|
||||
|
||||
print()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user