Merge pull request #32 from alexpolo1/tools/spellbook-mapper

tools: spellbook mapper — read skill binds from the game and check them against config

tools/map_spellbook.py

@@ -0,0 +1,362 @@
+"""
+Map the in-game skill-bind grid to config, using screenshots and clicks only.
+
+Opens the D2R skill-bind screen (default key `s`), walks every cell, and for
+each one:
+
+  1. hovers it and OCRs the tooltip  -> the skill's NAME
+  2. OCRs the icon's upper-right box -> the F-key it is bound to (if any)
+  3. saves the icon as a template with that corner REMOVED
+
+Then compares what the game actually has against the skill keys in config and
+prints the lines that disagree. Exits 1 on a mismatch so it can gate a run.
+
+    <botty-env-python> tools/map_spellbook.py            # map + report
+    <botty-env-python> tools/map_spellbook.py --assets   # also write templates
+    <botty-env-python> tools/map_spellbook.py --no-open  # grid already open
+
+WHY THE CORNER IS CROPPED
+-------------------------
+D2R draws the hotkey label ("F5", "F8") over the upper-right corner of the
+icon. An icon captured WITH that label only matches while the skill stays on
+that key — rebind it and the template silently stops matching, which looks like
+template rot rather than a bind change. So the corner is excluded from the
+saved template, and read separately as the binding. Same pixels, two purposes.
+
+WHY THIS EXISTS
+---------------
+Gear changes rebind skills. On 2026-08-28 an Enigma put Teleport on F5, which
+displaced Conviction, while config still said `conviction=f5` — so every
+attack-aura cast would have teleported the character mid-fight. The startup
+preflight did report it and it was dismissed as a marginal template. This tool
+answers the question directly instead of inferring it.
+"""
+
+import argparse
+import difflib
+import glob
+import os
+import re
+import sys
+import time
+import types
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src"))
+sys.modules.setdefault("discord", types.ModuleType("discord"))
+import ssl
+ssl.SSLContext.load_default_certs = lambda *a, **k: None
+
+import cv2
+
+import screen
+from config import Config
+from d2r_image.ocr import image_to_text
+from input_layer import keyboard, mouse
+from screen import convert_screen_to_monitor
+from utils.misc import focus_d2r_window
+
+# ── Grid geometry, measured against a 1280x720 client ────────────────────────
+CELL0 = (683, 410)      # centre of the top-left cell
+CELL_W = 46.5
+CELL_H = 45.0
+CELL_HALF = 24          # half-width of the icon box around a centre
+
+# The hotkey label box, relative to a cell's top-left corner (cell is
+# 2*CELL_HALF square). This is both the region blanked from saved templates and
+# the region searched for the binding.
+CORNER = (24, 2, 24, 16)          # x, y, w, h
+
+# The label is read by TEMPLATE MATCH, not OCR. It is ~20x12px of white glyph
+# over whatever icon happens to be behind it; brightness thresholding cannot
+# separate the two when the icon is also bright (F4's swirl, the row-4
+# weapons), and OCR managed only 4/8. A top-hat isolates small bright features
+# regardless of background, and the glyphs are a fixed UI font.
+BIND_TEMPLATE_DIR = os.path.join("assets", "templates", "skill_binds")
+BIND_MIN = 0.70
+# Search slack. Matching a same-sized crop is 1px-brittle: int() rounding in the
+# grid geometry lands a pixel off the measured centre and the score collapses
+# from ~1.0 to ~0.4. With slack, 8/8 matched at every threshold 0.60-0.78.
+BIND_PAD = 4
+
+# The tooltip renders directly ABOVE the hovered cell and grows upward, so its
+# position moves with the row. A fixed ROI reads the game world instead — the
+# first version of this tool did exactly that and returned "YEW Y" and "PET".
+TOOLTIP_BAND = (350, 1150, 280, 12)   # x0, x1, height_above, gap_above_cell
+
+# Identification is a fuzzy match against this list rather than a demand for
+# clean OCR. Real tooltip reads included "XI BLESSED HAMMER" and "HOLY SHIELD
+# L", and one frame OCR'd Fist of the Heavens as "LNMEPULE" on its first line
+# while the full text still contained the name.
+KNOWN_SKILLS = [
+    "BLESSED HAMMER", "CONCENTRATION", "CONVICTION", "FIST OF THE HEAVENS",
+    "HOLY BOLT", "HOLY SHIELD", "REDEMPTION", "VIGOR", "CLEANSING",
+    "TELEPORT", "VENGEANCE", "SALVATION", "DEFIANCE", "MIGHT", "PRAYER",
+    "TOME OF TOWN PORTAL", "TOME OF IDENTIFY", "OAK SAGE", "RAVEN",
+]
+MATCH_MIN = 0.72
+
+# The bind screen shows this hint along the bottom. It is how we tell whether
+# the grid is ALREADY open — pressing the key blindly toggles an open grid
+# CLOSED, which is exactly how the first live run came back with 0 skills.
+# Matched fuzzily: Tesseract reliably renders this as
+# "PRESS FI-F8 TO BIRD A SKIN" (BIND->BIRD, SKILL->SKIN), so an exact compare
+# never fires. Closed, the same ROI yields "" or "7," — the gap is wide.
+HINT_ROI = (450, 620, 400, 25)        # x, y, w, h
+HINT_TEXT = "PRESS F F TO BIND A SKILL"
+HINT_MIN = 0.55
+
+ASSET_DIR = os.path.join("assets", "templates", "skills_spellbook")
+
+# config key -> the name D2R shows in the tooltip
+CONFIG_SKILL_NAMES = {
+    "blessed_hammer": "BLESSED HAMMER",
+    "concentration": "CONCENTRATION",
+    "conviction": "CONVICTION",
+    "foh": "FIST OF THE HEAVENS",
+    "holy_bolt": "HOLY BOLT",
+    "holy_shield": "HOLY SHIELD",
+    "redemption": "REDEMPTION",
+    "vigor": "VIGOR",
+    "cleansing": "CLEANSING",
+    "teleport": "TELEPORT",
+}
+
+
+def grid_is_open(img=None) -> bool:
+    """True when the skill-bind grid is showing."""
+    img = screen.grab() if img is None else img
+    x, y, w, h = HINT_ROI
+    crop = cv2.resize(img[y:y + h, x:x + w], None, fx=2.5, fy=2.5)
+    text = re.sub(r"[^A-Z ]", " ", _ocr(crop, psm=7).upper())
+    text = re.sub(r"\s+", " ", text).strip()
+    if not text:
+        return False
+    return difflib.SequenceMatcher(None, HINT_TEXT, text).ratio() >= HINT_MIN
+
+
+def _cell_centre(col: int, row: int) -> tuple[int, int]:
+    return int(CELL0[0] + CELL_W * col), int(CELL0[1] + CELL_H * row)
+
+
+def _ocr(img, psm: int = 6) -> str:
+    if img is None or img.size == 0:
+        return ""
+    try:
+        res = image_to_text([img], psm=psm, crop_pad=False, invert=True, threshold=25)
+        return (res[0].text or "").strip() if res else ""
+    except Exception:
+        return ""
+
+
+def _isolate_glyph(bgr):
+    """Top-hat: keeps small bright features, drops whatever is behind them."""
+    g = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY).astype("float32")
+    k = cv2.getStructuringElement(cv2.MORPH_RECT, (9, 9))
+    th = cv2.morphologyEx(g, cv2.MORPH_TOPHAT, k)
+    return cv2.normalize(th, None, 0, 255, cv2.NORM_MINMAX).astype("uint8")
+
+
+_BIND_TEMPLATES: dict | None = None
+
+
+def _bind_templates() -> dict:
+    global _BIND_TEMPLATES
+    if _BIND_TEMPLATES is None:
+        _BIND_TEMPLATES = {}
+        for path in glob.glob(os.path.join(BIND_TEMPLATE_DIR, "*.png")):
+            im = cv2.imread(path)
+            if im is not None:
+                _BIND_TEMPLATES[os.path.basename(path)[:-4].lower()] = _isolate_glyph(im)
+    return _BIND_TEMPLATES
+
+
+def read_bound_key(img, cx: int, cy: int) -> str | None:
+    """Template-match the hotkey label -> 'f5', or None when unbound."""
+    x, y, w, h = CORNER
+    x0 = cx - CELL_HALF + x
+    y0 = cy - CELL_HALF + y
+    win = img[y0 - BIND_PAD:y0 + h + BIND_PAD, x0 - BIND_PAD:x0 + w + BIND_PAD]
+    if win.size == 0 or win.shape[0] < 10:
+        return None
+    g = _isolate_glyph(win)
+    best, score = None, 0.0
+    for name, tpl in _bind_templates().items():
+        if g.shape[0] < tpl.shape[0] or g.shape[1] < tpl.shape[1]:
+            continue
+        s = cv2.matchTemplate(g, tpl, cv2.TM_CCOEFF_NORMED).max()
+        if s > score:
+            best, score = name, s
+    return best if score >= BIND_MIN else None
+
+
+def icon_without_corner(cell_img):
+    """The icon with the hotkey label blanked — safe to store as a template."""
+    icon = cell_img.copy()
+    x, y, w, h = CORNER
+    icon[y:y + h, x:x + w] = 0
+    return icon
+
+
+def read_skill_name(img, cy: int) -> str:
+    """Identify the hovered skill from its tooltip, fuzzily."""
+    x0, x1, above, gap = TOOLTIP_BAND
+    band = img[max(0, cy - above):max(1, cy - gap), x0:x1]
+    text = re.sub(r"[^A-Z ]", " ", _ocr(band, psm=6).upper())
+    text = re.sub(r"\s+", " ", text).strip()
+    if not text:
+        return ""
+    for known in KNOWN_SKILLS:              # exact substring is decisive
+        if known in text:
+            return known
+    best, score, words = "", 0.0, text.split()
+    for known in KNOWN_SKILLS:              # else slide a window of the right length
+        kw = known.split()
+        for i in range(max(1, len(words) - len(kw) + 1)):
+            cand = " ".join(words[i:i + len(kw)])
+            r = difflib.SequenceMatcher(None, known, cand).ratio()
+            if r > score:
+                best, score = known, r
+    return best if score >= MATCH_MIN else ""
+
+
+def map_grid(cols: int, rows: int, save_assets: bool) -> list[dict]:
+    found = []
+    if save_assets:
+        os.makedirs(ASSET_DIR, exist_ok=True)
+
+    for row in range(rows):
+        for col in range(cols):
+            cx, cy = _cell_centre(col, row)
+            mx, my = convert_screen_to_monitor((cx, cy))
+            mouse.move(mx, my, randomize=0, delay_factor=[0.25, 0.35])
+            time.sleep(0.55)
+            img = screen.grab()
+
+            name = read_skill_name(img, cy)
+            if not name:
+                continue                    # empty cell — no tooltip appeared
+            cell = img[cy - CELL_HALF:cy + CELL_HALF, cx - CELL_HALF:cx + CELL_HALF]
+            key = read_bound_key(img, cx, cy)
+            found.append({"name": name, "key": key, "col": col, "row": row})
+            print(f"  ({col},{row})  {name:<24} {key or '-'}", flush=True)
+
+            if save_assets:
+                slug = re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_")
+                cv2.imwrite(os.path.join(ASSET_DIR, f"{slug}.png"),
+                            icon_without_corner(cell))
+    return found
+
+
+def compare_with_config(found: list[dict]) -> int:
+    cfg = Config()
+    char_type = str(cfg.char.get("type", "")).lower()
+    section = getattr(cfg, char_type, {}) or {}
+    configured = {k: str(v).strip().lower()
+                  for k, v in section.items() if str(v).strip()}
+    tp = str(cfg.char.get("teleport", "")).strip().lower()
+    if tp:
+        configured["teleport"] = tp
+
+    in_game = {d["name"]: d["key"] for d in found if d["key"]}
+    problems = 0
+
+    print("\n  config skill        expects   game has          verdict")
+    print("  " + "-" * 66)
+    for skill, key in sorted(configured.items()):
+        want = CONFIG_SKILL_NAMES.get(skill)
+        actual = next((n for n, k in in_game.items() if k == key), None)
+        if want and actual == want:
+            verdict = "ok"
+        elif actual is None:
+            verdict = "nothing bound to that key"
+            problems += 1
+        else:
+            verdict = f"MISMATCH -> that key casts {actual}"
+            problems += 1
+        print(f"  {skill:<19} {key:<9} {actual or '-':<17} {verdict}")
+
+    extra = [(s, in_game[n]) for s, n in CONFIG_SKILL_NAMES.items()
+             if n in in_game and s not in configured]
+    if extra:
+        print("\n  Bound in game but not used by config:")
+        for s, k in extra:
+            print(f"    {s:<19} is on {k}")
+
+    if problems:
+        print("\n  Config lines matching what the game actually has:")
+        for name, key in sorted(in_game.items()):
+            slug = next((s for s, n in CONFIG_SKILL_NAMES.items() if n == name), None)
+            if slug:
+                print(f"    {slug}={key}")
+    return problems
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--cols", type=int, default=6)
+    ap.add_argument("--rows", type=int, default=5)
+    ap.add_argument("--key", default="s", help="key that opens the bind grid")
+    ap.add_argument("--assets", action="store_true",
+                    help="save corner-cropped icon templates")
+    ap.add_argument("--no-open", action="store_true",
+                    help="grid is already open; do not press the key")
+    args = ap.parse_args()
+
+    screen.start_detecting_window()
+    screen.find_and_set_window_position(force=True)
+    if not focus_d2r_window():
+        print("map_spellbook: could not focus D2R — is it running?")
+        return 2
+
+    time.sleep(0.5)
+    # Detect, do not assume. Toggling an already-open grid closes it.
+    opened_by_us = False
+    if not grid_is_open():
+        if args.no_open:
+            print("map_spellbook: --no-open was given but the bind grid is not "
+                  "open. Open it and re-run, or drop --no-open.")
+            return 2
+        keyboard.send(args.key)
+        time.sleep(1.0)
+        opened_by_us = True
+        if not grid_is_open():
+            print(f"map_spellbook: pressed {args.key!r} but the bind grid did not "
+                  f"open. Use --key <k> if your bind screen is on another key.")
+            return 2
+    else:
+        print("Bind grid already open — leaving it as found.")
+
+    print(f"Scanning {args.cols}x{args.rows} bind grid...", flush=True)
+    found = map_grid(args.cols, args.rows, args.assets)
+    bound = sum(1 for d in found if d["key"])
+
+    # A tool that cannot read the screen must say so, not report its blindness
+    # as findings. The first version scanned a closed grid, identified nothing,
+    # and then confidently declared all seven configured keys unbound.
+    if len(found) < 3 or bound == 0:
+        print(f"\nmap_spellbook: read {len(found)} skill(s), {bound} bound — that is "
+              f"too few to trust, so NO comparison was made.\n"
+              f"  - is the bind grid actually open? (try --key <k>, or open it "
+              f"yourself and pass --no-open)\n"
+              f"  - if your client is not 1280x720, the grid geometry at the top "
+              f"of this file needs re-measuring.")
+        return 2
+
+    print(f"\nFound {len(found)} skills, {bound} of them bound.")
+    problems = compare_with_config(found)
+
+    if opened_by_us:
+        keyboard.send(args.key)      # close it only if we opened it
+
+    if problems:
+        print(f"\n{problems} mismatch(es). Fix the binds in game or the keys in "
+              f"config/profiles/<profile>/profile.ini before running — a wrong key "
+              f"means the bot casts the wrong skill.")
+        return 1
+    print("\nAll configured skill keys match the game.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/testbed.py

@@ -358,6 +358,7 @@ def main():
     scenarios = {
         "binds": lambda: _run_tool("set_binds_from_params.py"),
         "capture": lambda: _run_tool("capture_skill_hotkeys.py"),
+        "spellbook": lambda: _run_tool("map_spellbook.py", *sys.argv[2:]),
         "audit": lambda: _run_tool("audit_stash_pickit.py"),
         "keyo": lambda: _run_tool("set_controls_keyo.py", *sys.argv[2:]),
         "gems": _gems,