my-botty/tools/hermes.py

"""
Hermes — vision-capable LLM agent for live D2R game analysis and bot development.

Grabs the current D2R window, sends it to the local Qwen vision model, and
lets you ask questions or request feature ideas based on what's on screen.

Usage:
    <botty-env-python> tools/hermes.py [initial prompt]
    <botty-env-python> tools/hermes.py              # interactive mode

The LLM sees the exact screenshot the bot would see (same grab() call, same
resolution). Useful for:
  - Understanding why a template match failed ("what does the NPC look like?")
  - Prototyping feature logic ("what info is visible on the stash screen?")
  - Checking item tooltips without running the full bot
  - Building new detection templates from live game state

Controls (interactive mode):
  Enter        send current message (re-grabs screen each turn)
  g / grab     force a fresh grab and print its size before next query
  q / quit     exit
"""

import os
import sys
import json
import base64
import io
import urllib.request
import urllib.error
import time
import textwrap

TOOLS = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(TOOLS)
SRC = os.path.join(ROOT, "src")
if SRC not in sys.path:
    sys.path.insert(0, SRC)

HERMES_URL = "http://192.168.1.98:8010/v1/chat/completions"
HERMES_MODEL = "qwen3.6-27b-autoround"
MAX_TOKENS = 1024
IMG_WIDTH = 1280   # resize to keep payload manageable
IMG_QUALITY = 85   # JPEG quality for the grab


def _patch_ssl():
    import ssl
    _orig = ssl.SSLContext.load_default_certs
    def _safe(self, purpose=ssl.Purpose.CLIENT_AUTH):
        try:
            _orig(self, purpose)
        except ssl.SSLError:
            try:
                import certifi
                self.load_verify_locations(certifi.where())
            except Exception:
                pass
    ssl.SSLContext.load_default_certs = _safe


_SYSTEM = textwrap.dedent("""\
    You are Hermes, a vision-capable assistant for the my-botty Diablo II: Resurrected
    automation project. The bot controls a Hammerdin character in Hell difficulty.

    COORDINATE SYSTEMS (for feature/template development):
    - Screen: D2R client area top-left = (0,0), size 1280×720
    - Left panel (stash/NPC): ROI roughly x=33..415, y=84..466, 10×10 grid, 38×38px slots
    - Right panel (character inventory): x=866..1247, y=348..535, 10×4 grid, 38×38px slots
    - Belt: y≈540, row of 4 potion columns
    - Minimap: top-right corner ~x=1050..1270, y=0..130
    - Health orb: bottom-left ~x=0..120, y=560..720
    - Mana orb: bottom-right ~x=1160..1280, y=560..720
    - WP/stash UI open indicator: gold button visible at ~(95,140) (stash) or (684,140) (inv)

    TEMPLATE MATCHING: src/template_finder.py uses OpenCV matchTemplate. Templates live in
    assets/templates/. New templates should be ~30×30px crops of the target UI element at
    1280×720 resolution.

    INPUT LAYER: src/input_layer/win_input.py — SetCursorPos + zero-delta MOUSEEVENTF_MOVE
    for accurate positioning (Win11 pointer acceleration bypass). All clicks go through
    input_layer.mouse.

    When suggesting code, use the existing patterns from the codebase:
    - Grabbing: from screen import grab; img = grab(True)  [True = monitor coords]
    - Template: template_finder.search(["TEMPLATE_NAME"], img, threshold=0.7)
    - Mouse: from input_layer import mouse; mouse.click(x, y)
    - Wait: from utils.misc import wait; wait(0.1, 0.2)
    - Logging: from logger import Logger; Logger.info("msg")
""")


def _grab_b64() -> tuple[str, tuple[int, int]]:
    """Grab D2R screen and return (base64_jpeg, (w, h))."""
    import ctypes
    ctypes.windll.user32.SetProcessDPIAware()
    from screen import grab, start_detecting_window, stop_detecting_window
    from PIL import Image
    import numpy as np

    start_detecting_window()
    try:
        img_np = grab(True)
    finally:
        stop_detecting_window()

    img_pil = Image.fromarray(img_np[..., ::-1])  # BGR -> RGB
    orig_w, orig_h = img_pil.size
    if orig_w > IMG_WIDTH:
        scale = IMG_WIDTH / orig_w
        img_pil = img_pil.resize((IMG_WIDTH, int(orig_h * scale)), Image.LANCZOS)

    buf = io.BytesIO()
    img_pil.save(buf, format="JPEG", quality=IMG_QUALITY)
    b64 = base64.b64encode(buf.getvalue()).decode()
    return b64, img_pil.size


def _ask(messages: list, img_b64: str) -> str:
    """Send conversation + current screen to Hermes; return reply text."""
    # Inject the screenshot as the last user turn's image
    user_content = [
        {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}},
    ]
    if messages and messages[-1]["role"] == "user":
        last_text = messages[-1]["content"]
        user_content.append({"type": "text", "text": last_text})
        send_messages = messages[:-1] + [{"role": "user", "content": user_content}]
    else:
        send_messages = messages + [{"role": "user", "content": user_content}]

    payload = json.dumps({
        "model": HERMES_MODEL,
        "max_tokens": MAX_TOKENS,
        "temperature": 0.3,
        "repetition_penalty": 1.15,
        "chat_template_kwargs": {"enable_thinking": False},
        "messages": [{"role": "system", "content": _SYSTEM}] + send_messages,
    }).encode()

    req = urllib.request.Request(
        HERMES_URL,
        data=payload,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    try:
        r = urllib.request.urlopen(req, timeout=60)
        data = json.loads(r.read())
        return data["choices"][0]["message"]["content"].strip()
    except urllib.error.HTTPError as e:
        body = e.read().decode()
        return f"[HTTP {e.code}] {body[:300]}"
    except Exception as e:
        return f"[Error] {e}"


def _print_wrapped(text: str, width: int = 100):
    import sys
    out = sys.stdout
    for line in text.splitlines():
        filled = line if len(line) <= width else textwrap.fill(line, width=width, subsequent_indent="  ")
        try:
            print(filled)
        except UnicodeEncodeError:
            print(filled.encode("ascii", errors="replace").decode("ascii"))


def main():
    _patch_ssl()

    print("Hermes — D2R vision agent  (q=quit, g=grab)\n")

    initial = " ".join(sys.argv[1:]).strip()
    history: list[dict] = []
    img_b64 = None
    img_size = None

    def do_grab():
        nonlocal img_b64, img_size
        print("Grabbing D2R screen...", end=" ", flush=True)
        t0 = time.time()
        img_b64, img_size = _grab_b64()
        print(f"{img_size[0]}×{img_size[1]}  ({len(img_b64)//1024}KB)  [{time.time()-t0:.1f}s]")

    do_grab()

    def send(prompt: str):
        history.append({"role": "user", "content": prompt})
        print("\nHermes: ", end="", flush=True)
        t0 = time.time()
        reply = _ask(history, img_b64)
        history.append({"role": "assistant", "content": reply})
        print()
        _print_wrapped(reply)
        print(f"\n[{time.time()-t0:.1f}s]\n")

    if initial:
        send(initial)

    while True:
        try:
            raw = input("You: ").strip()
        except (EOFError, KeyboardInterrupt):
            print("\nbye")
            break
        if not raw:
            continue
        if raw.lower() in ("q", "quit", "exit"):
            break
        if raw.lower() in ("g", "grab"):
            do_grab()
            continue
        # Auto-regrab each turn so Hermes always sees fresh state
        do_grab()
        send(raw)


if __name__ == "__main__":
    main()