diff --git a/botty_next/__init__.py b/botty_next/__init__.py new file mode 100644 index 0000000..3a58fcd --- /dev/null +++ b/botty_next/__init__.py @@ -0,0 +1,5 @@ +"""Botty Next offline-first visual QA harness.""" + +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/botty_next/capture/__init__.py b/botty_next/capture/__init__.py new file mode 100644 index 0000000..773477c --- /dev/null +++ b/botty_next/capture/__init__.py @@ -0,0 +1,6 @@ +"""Capture backends for offline fixtures and live observer mode.""" + +from botty_next.capture.mss_backend import MssCaptureBackend +from botty_next.capture.window import WindowRegion, find_window_region + +__all__ = ["MssCaptureBackend", "WindowRegion", "find_window_region"] diff --git a/botty_next/capture/mss_backend.py b/botty_next/capture/mss_backend.py new file mode 100644 index 0000000..a2fe83d --- /dev/null +++ b/botty_next/capture/mss_backend.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from pathlib import Path + +import cv2 +import mss +import numpy as np + +from botty_next.capture.window import WindowRegion + + +class MssCaptureBackend: + def grab(self, region: WindowRegion | None = None) -> np.ndarray: + with mss.mss() as screen_capture: + monitor = region.as_mss_monitor() if region else screen_capture.monitors[1] + shot = screen_capture.grab(monitor) + + bgra = np.asarray(shot) + return cv2.cvtColor(bgra, cv2.COLOR_BGRA2BGR) + + +def save_frame(frame: np.ndarray, output_path: str | Path) -> Path: + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + if not cv2.imwrite(str(output), frame): + raise RuntimeError(f"failed to write screenshot: {output}") + return output diff --git a/botty_next/capture/window.py b/botty_next/capture/window.py new file mode 100644 index 0000000..ddfa084 --- /dev/null +++ b/botty_next/capture/window.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class WindowRegion: + left: int + top: int + width: int + height: int + title: str + + def as_mss_monitor(self) -> dict[str, int]: + return { + "left": self.left, + "top": self.top, + "width": self.width, + "height": self.height, + } + + +def find_window_region(title_contains: str) -> WindowRegion: + import win32gui + + matches: list[WindowRegion] = [] + + def collect(hwnd: int, _extra) -> bool: + if not win32gui.IsWindowVisible(hwnd): + return True + + title = win32gui.GetWindowText(hwnd) + if title_contains.lower() not in title.lower(): + return True + + left, top, right, bottom = win32gui.GetWindowRect(hwnd) + width = right - left + height = bottom - top + if width > 0 and height > 0: + matches.append(WindowRegion(left, top, width, height, title)) + return True + + win32gui.EnumWindows(collect, None) + if not matches: + raise RuntimeError(f"no visible window found containing title: {title_contains}") + + return max(matches, key=lambda region: region.width * region.height) diff --git a/botty_next/cli.py b/botty_next/cli.py new file mode 100644 index 0000000..450842f --- /dev/null +++ b/botty_next/cli.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from botty_next.capture.mss_backend import MssCaptureBackend, save_frame +from botty_next.capture.window import find_window_region +from botty_next.config import load_config +from botty_next.vision.fixtures import load_image +from botty_next.vision.ocr import run_tesseract_ocr, save_ocr_preprocess_debug +from botty_next.vision.template_matching import match_template, save_match_debug + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="botty-next") + subparsers = parser.add_subparsers(dest="command", required=True) + + config_parser = subparsers.add_parser("config") + config_subparsers = config_parser.add_subparsers(dest="config_command", required=True) + validate_parser = config_subparsers.add_parser("validate") + validate_parser.add_argument("-c", "--config", required=True, type=Path) + validate_parser.set_defaults(handler=validate_config) + + detect_parser = subparsers.add_parser("detect") + detect_parser.add_argument("detector", choices=["template"], help="detector to run") + detect_parser.add_argument("--image", required=True, type=Path) + detect_parser.add_argument("--template", required=True, type=Path) + detect_parser.add_argument("--threshold", type=float, default=0.85) + detect_parser.add_argument("--debug-output", type=Path) + detect_parser.set_defaults(handler=detect) + + capture_parser = subparsers.add_parser("capture") + capture_parser.add_argument("--output", required=True, type=Path) + capture_parser.add_argument("--window-title", type=str) + capture_parser.set_defaults(handler=capture) + + ocr_parser = subparsers.add_parser("ocr") + ocr_parser.add_argument("--image", required=True, type=Path) + ocr_parser.add_argument("--lang", default="eng") + ocr_parser.add_argument("--psm", type=int, default=6) + ocr_parser.add_argument("--tesseract-cmd") + ocr_parser.add_argument("--debug-output", type=Path) + ocr_parser.set_defaults(handler=ocr) + + return parser + + +def validate_config(args: argparse.Namespace) -> int: + config = load_config(args.config) + print(json.dumps(config.model_dump(mode="json"), indent=2)) + return 0 + + +def detect(args: argparse.Namespace) -> int: + image = load_image(args.image) + template = load_image(args.template) + result = match_template(image, template, threshold=args.threshold) + + if args.debug_output: + save_match_debug(image, result, args.debug_output) + + print(json.dumps(_result_to_dict(result), indent=2)) + return 0 if result.passed else 1 + + +def capture(args: argparse.Namespace) -> int: + region = find_window_region(args.window_title) if args.window_title else None + frame = MssCaptureBackend().grab(region) + output = save_frame(frame, args.output) + + print( + json.dumps( + { + "output": str(output), + "shape": tuple(map(int, frame.shape)), + "window": region.title if region else None, + }, + indent=2, + ) + ) + return 0 + + +def ocr(args: argparse.Namespace) -> int: + image = load_image(args.image) + if args.debug_output: + save_ocr_preprocess_debug(image, args.debug_output) + + try: + result = run_tesseract_ocr( + image, + lang=args.lang, + psm=args.psm, + tesseract_cmd=args.tesseract_cmd, + ) + except RuntimeError as exc: + print( + json.dumps( + { + "error": str(exc), + "debug_output": str(args.debug_output) if args.debug_output else None, + }, + indent=2, + ) + ) + return 2 + + print(json.dumps(_ocr_result_to_dict(result), indent=2)) + return 0 + + +def _result_to_dict(result) -> dict: + return { + "confidence": result.confidence, + "bbox": result.bbox, + "passed": result.passed, + "method": result.method, + "debug": result.debug, + } + + +def _ocr_result_to_dict(result) -> dict: + return { + "text": result.text, + "confidence": result.confidence, + "bbox": result.bbox, + "debug": result.debug, + } + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + return args.handler(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/botty_next/config/__init__.py b/botty_next/config/__init__.py new file mode 100644 index 0000000..4311695 --- /dev/null +++ b/botty_next/config/__init__.py @@ -0,0 +1,5 @@ +"""Configuration loading and validation.""" + +from botty_next.config.models import BottyNextConfig, load_config + +__all__ = ["BottyNextConfig", "load_config"] diff --git a/botty_next/config/default.yaml b/botty_next/config/default.yaml new file mode 100644 index 0000000..3017805 --- /dev/null +++ b/botty_next/config/default.yaml @@ -0,0 +1,13 @@ +profile_name: local +capture: + backend: fixture + monitor: 1 + fps_limit: 10 + window_title: null +vision: + template_threshold: 0.85 + debug_output_dir: botty_next/debug/output +input: + enabled: false + dry_run: true + emergency_stop_key: f12 diff --git a/botty_next/config/models.py b/botty_next/config/models.py new file mode 100644 index 0000000..63f775a --- /dev/null +++ b/botty_next/config/models.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class CaptureConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + backend: Literal["fixture", "mss", "dxcam"] = "fixture" + monitor: int = 1 + fps_limit: int = Field(default=10, ge=1, le=240) + window_title: str | None = None + + +class VisionConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + template_threshold: float = Field(default=0.85, ge=0.0, le=1.0) + debug_output_dir: Path = Path("botty_next/debug/output") + + +class InputConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + enabled: bool = False + dry_run: bool = True + emergency_stop_key: str = "f12" + + @field_validator("dry_run") + @classmethod + def dry_run_required_when_disabled(cls, value: bool) -> bool: + return value + + +class BottyNextConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + profile_name: str = "local" + capture: CaptureConfig = Field(default_factory=CaptureConfig) + vision: VisionConfig = Field(default_factory=VisionConfig) + input: InputConfig = Field(default_factory=InputConfig) + + @field_validator("input") + @classmethod + def input_must_be_explicit_and_dry_run_by_default(cls, value: InputConfig) -> InputConfig: + if value.enabled and value.dry_run is False: + raise ValueError("live input cannot be enabled without a future explicit safety gate") + return value + + +def load_config(path: str | Path) -> BottyNextConfig: + config_path = Path(path) + with config_path.open("r", encoding="utf-8") as handle: + raw = yaml.safe_load(handle) or {} + return BottyNextConfig.model_validate(raw) diff --git a/botty_next/debug/__init__.py b/botty_next/debug/__init__.py new file mode 100644 index 0000000..71deadb --- /dev/null +++ b/botty_next/debug/__init__.py @@ -0,0 +1 @@ +"""Debug image and report output helpers.""" diff --git a/botty_next/input/__init__.py b/botty_next/input/__init__.py new file mode 100644 index 0000000..03c9b60 --- /dev/null +++ b/botty_next/input/__init__.py @@ -0,0 +1,4 @@ +"""Input abstraction layer. + +Live input is intentionally not implemented in the bootstrap harness. +""" diff --git a/botty_next/routines/__init__.py b/botty_next/routines/__init__.py new file mode 100644 index 0000000..90997dc --- /dev/null +++ b/botty_next/routines/__init__.py @@ -0,0 +1 @@ +"""Offline/private routine replay harness.""" diff --git a/botty_next/state/__init__.py b/botty_next/state/__init__.py new file mode 100644 index 0000000..530eb30 --- /dev/null +++ b/botty_next/state/__init__.py @@ -0,0 +1 @@ +"""State detection and transition logic.""" diff --git a/botty_next/tests/__init__.py b/botty_next/tests/__init__.py new file mode 100644 index 0000000..ca21b65 --- /dev/null +++ b/botty_next/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the Botty Next harness.""" diff --git a/botty_next/tests/test_capture.py b/botty_next/tests/test_capture.py new file mode 100644 index 0000000..45f82d5 --- /dev/null +++ b/botty_next/tests/test_capture.py @@ -0,0 +1,12 @@ +from botty_next.capture.window import WindowRegion + + +def test_window_region_converts_to_mss_monitor() -> None: + region = WindowRegion(left=10, top=20, width=640, height=480, title="Example") + + assert region.as_mss_monitor() == { + "left": 10, + "top": 20, + "width": 640, + "height": 480, + } diff --git a/botty_next/tests/test_cli.py b/botty_next/tests/test_cli.py new file mode 100644 index 0000000..e338d77 --- /dev/null +++ b/botty_next/tests/test_cli.py @@ -0,0 +1,28 @@ +from botty_next.cli import main + + +def test_config_validate_cli_starts(capsys) -> None: + exit_code = main(["config", "validate", "-c", "botty_next/config/default.yaml"]) + + captured = capsys.readouterr() + assert exit_code == 0 + assert '"profile_name": "local"' in captured.out + + +def test_detect_cli_runs_template_detector(capsys) -> None: + exit_code = main( + [ + "detect", + "template", + "--image", + "fixtures/screenshots/sample_scene.ppm", + "--template", + "fixtures/templates/sample_marker.ppm", + "--threshold", + "0.99", + ] + ) + + captured = capsys.readouterr() + assert exit_code == 0 + assert '"passed": true' in captured.out diff --git a/botty_next/tests/test_config.py b/botty_next/tests/test_config.py new file mode 100644 index 0000000..1b19e70 --- /dev/null +++ b/botty_next/tests/test_config.py @@ -0,0 +1,10 @@ +from botty_next.config import load_config + + +def test_load_default_config() -> None: + config = load_config("botty_next/config/default.yaml") + + assert config.profile_name == "local" + assert config.capture.backend == "fixture" + assert config.input.enabled is False + assert config.input.dry_run is True diff --git a/botty_next/tests/test_fixtures.py b/botty_next/tests/test_fixtures.py new file mode 100644 index 0000000..fee40c0 --- /dev/null +++ b/botty_next/tests/test_fixtures.py @@ -0,0 +1,13 @@ +from botty_next.vision.fixtures import load_screenshot, load_template + + +def test_load_sample_screenshot_fixture() -> None: + image = load_screenshot("sample_scene.ppm") + + assert image.shape == (8, 8, 3) + + +def test_load_sample_template_fixture() -> None: + template = load_template("sample_marker.ppm") + + assert template.shape == (3, 3, 3) diff --git a/botty_next/tests/test_ocr.py b/botty_next/tests/test_ocr.py new file mode 100644 index 0000000..791d9ab --- /dev/null +++ b/botty_next/tests/test_ocr.py @@ -0,0 +1,42 @@ +import sys +from types import SimpleNamespace + +import pytest + +from botty_next.vision.fixtures import load_screenshot +from botty_next.vision.ocr import preprocess_for_ocr, run_tesseract_ocr, save_ocr_preprocess_debug + + +def test_preprocess_for_ocr_returns_thresholded_image() -> None: + image = load_screenshot("sample_scene.ppm") + + processed = preprocess_for_ocr(image) + + assert processed.ndim == 2 + assert processed.shape == (16, 16) + + +def test_save_ocr_preprocess_debug(tmp_path) -> None: + image = load_screenshot("sample_scene.ppm") + + output = save_ocr_preprocess_debug(image, tmp_path / "ocr.png") + + assert output.exists() + + +def test_run_tesseract_ocr_uses_pytesseract_adapter(monkeypatch) -> None: + fake = SimpleNamespace( + Output=SimpleNamespace(DICT="dict"), + pytesseract=SimpleNamespace(tesseract_cmd=None), + image_to_string=lambda *_args, **_kwargs: "Short Sword\n", + image_to_data=lambda *_args, **_kwargs: {"conf": ["95", "-1", "85"]}, + ) + monkeypatch.setitem(sys.modules, "pytesseract", fake) + + image = load_screenshot("sample_scene.ppm") + result = run_tesseract_ocr(image, tesseract_cmd="C:/Tesseract/tesseract.exe") + + assert result.text == "Short Sword" + assert result.confidence == pytest.approx(0.9) + assert result.debug["backend"] == "pytesseract" + assert fake.pytesseract.tesseract_cmd == "C:/Tesseract/tesseract.exe" diff --git a/botty_next/tests/test_template_matching.py b/botty_next/tests/test_template_matching.py new file mode 100644 index 0000000..9413090 --- /dev/null +++ b/botty_next/tests/test_template_matching.py @@ -0,0 +1,24 @@ +from botty_next.vision.fixtures import load_screenshot, load_template +from botty_next.vision.template_matching import match_template, save_match_debug + + +def test_template_match_finds_sample_marker() -> None: + image = load_screenshot("sample_scene.ppm") + template = load_template("sample_marker.ppm") + + result = match_template(image, template, threshold=0.99) + + assert result.passed is True + assert result.confidence >= 0.99 + assert result.bbox == (3, 2, 3, 3) + assert "image_shape" in result.debug + + +def test_template_match_can_save_debug_image(tmp_path) -> None: + image = load_screenshot("sample_scene.ppm") + template = load_template("sample_marker.ppm") + result = match_template(image, template, threshold=0.99) + + output = save_match_debug(image, result, tmp_path / "marked.png") + + assert output.exists() diff --git a/botty_next/vision/__init__.py b/botty_next/vision/__init__.py new file mode 100644 index 0000000..4a43697 --- /dev/null +++ b/botty_next/vision/__init__.py @@ -0,0 +1,6 @@ +"""Vision helpers and detectors.""" + +from botty_next.vision.ocr import OcrResult, preprocess_for_ocr, run_tesseract_ocr +from botty_next.vision.template_matching import MatchResult, match_template + +__all__ = ["MatchResult", "OcrResult", "match_template", "preprocess_for_ocr", "run_tesseract_ocr"] diff --git a/botty_next/vision/fixtures.py b/botty_next/vision/fixtures.py new file mode 100644 index 0000000..e94a0b7 --- /dev/null +++ b/botty_next/vision/fixtures.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from pathlib import Path + +import cv2 +import numpy as np + + +def load_image(path: str | Path, *, grayscale: bool = False) -> np.ndarray: + image_path = Path(path) + if not image_path.exists(): + raise FileNotFoundError(f"image fixture does not exist: {image_path}") + + flag = cv2.IMREAD_GRAYSCALE if grayscale else cv2.IMREAD_COLOR + image = cv2.imread(str(image_path), flag) + if image is None: + raise ValueError(f"OpenCV could not read image fixture: {image_path}") + return image + + +def load_screenshot(name: str, root: str | Path = "fixtures/screenshots") -> np.ndarray: + return load_image(Path(root) / name) + + +def load_template(name: str, root: str | Path = "fixtures/templates") -> np.ndarray: + return load_image(Path(root) / name) diff --git a/botty_next/vision/ocr.py b/botty_next/vision/ocr.py new file mode 100644 index 0000000..dc6c87a --- /dev/null +++ b/botty_next/vision/ocr.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from dataclasses import dataclass +from importlib import import_module +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np + + +@dataclass(frozen=True) +class OcrResult: + text: str + confidence: float + bbox: tuple[int, int, int, int] | None + debug: dict[str, Any] + + +def preprocess_for_ocr(image: np.ndarray, *, scale: float = 2.0) -> np.ndarray: + if image.size == 0: + raise ValueError("image is empty") + + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) if image.ndim == 3 else image + if scale != 1.0: + gray = cv2.resize(gray, None, fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) + denoised = cv2.GaussianBlur(gray, (3, 3), 0) + return cv2.adaptiveThreshold( + denoised, + 255, + cv2.ADAPTIVE_THRESH_GAUSSIAN_C, + cv2.THRESH_BINARY, + 31, + 7, + ) + + +def run_tesseract_ocr( + image: np.ndarray, + *, + lang: str = "eng", + psm: int = 6, + tesseract_cmd: str | None = None, +) -> OcrResult: + try: + pytesseract = import_module("pytesseract") + except ModuleNotFoundError as exc: + raise RuntimeError( + "pytesseract is not installed; run install.bat or install requirements.txt" + ) from exc + + if tesseract_cmd: + pytesseract.pytesseract.tesseract_cmd = tesseract_cmd + + processed = preprocess_for_ocr(image) + config = f"--psm {psm}" + text = pytesseract.image_to_string(processed, lang=lang, config=config).strip() + confidences = _read_confidences( + pytesseract.image_to_data( + processed, + lang=lang, + config=config, + output_type=pytesseract.Output.DICT, + ) + ) + confidence = sum(confidences) / len(confidences) if confidences else 0.0 + return OcrResult( + text=text, + confidence=confidence, + bbox=None, + debug={ + "backend": "pytesseract", + "lang": lang, + "psm": psm, + "preprocessed_shape": tuple(map(int, processed.shape)), + "word_confidences": confidences, + }, + ) + + +def save_ocr_preprocess_debug(image: np.ndarray, output_path: str | Path) -> Path: + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + if not cv2.imwrite(str(output), preprocess_for_ocr(image)): + raise RuntimeError(f"failed to write OCR debug image: {output}") + return output + + +def _read_confidences(data: dict[str, list[Any]]) -> list[float]: + values: list[float] = [] + for raw in data.get("conf", []): + try: + confidence = float(raw) + except (TypeError, ValueError): + continue + if confidence >= 0: + values.append(confidence / 100.0) + return values diff --git a/botty_next/vision/template_matching.py b/botty_next/vision/template_matching.py new file mode 100644 index 0000000..27b41b3 --- /dev/null +++ b/botty_next/vision/template_matching.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import cv2 +import numpy as np + + +@dataclass(frozen=True) +class MatchResult: + confidence: float + bbox: tuple[int, int, int, int] + passed: bool + method: str + debug: dict[str, Any] + + +def _as_gray(image: np.ndarray) -> np.ndarray: + if image.ndim == 2: + return image + return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + + +def match_template( + image: np.ndarray, + template: np.ndarray, + *, + threshold: float = 0.85, + method: int = cv2.TM_CCOEFF_NORMED, +) -> MatchResult: + if image.size == 0: + raise ValueError("image is empty") + if template.size == 0: + raise ValueError("template is empty") + if template.shape[0] > image.shape[0] or template.shape[1] > image.shape[1]: + raise ValueError("template cannot be larger than image") + + image_gray = _as_gray(image) + template_gray = _as_gray(template) + response = cv2.matchTemplate(image_gray, template_gray, method) + min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(response) + + if method in (cv2.TM_SQDIFF, cv2.TM_SQDIFF_NORMED): + top_left = min_loc + confidence = 1.0 - float(min_val) + else: + top_left = max_loc + confidence = float(max_val) + + width = int(template.shape[1]) + height = int(template.shape[0]) + bbox = (int(top_left[0]), int(top_left[1]), width, height) + return MatchResult( + confidence=confidence, + bbox=bbox, + passed=confidence >= threshold, + method=_method_name(method), + debug={ + "threshold": threshold, + "min_value": float(min_val), + "max_value": float(max_val), + "min_location": tuple(map(int, min_loc)), + "max_location": tuple(map(int, max_loc)), + "image_shape": tuple(map(int, image.shape)), + "template_shape": tuple(map(int, template.shape)), + }, + ) + + +def save_match_debug(image: np.ndarray, result: MatchResult, output_path: str | Path) -> Path: + output = Path(output_path) + output.parent.mkdir(parents=True, exist_ok=True) + + marked = image.copy() + x, y, width, height = result.bbox + color = (0, 255, 0) if result.passed else (0, 0, 255) + cv2.rectangle(marked, (x, y), (x + width, y + height), color, 2) + cv2.imwrite(str(output), marked) + return output + + +def _method_name(method: int) -> str: + names = { + cv2.TM_CCOEFF: "TM_CCOEFF", + cv2.TM_CCOEFF_NORMED: "TM_CCOEFF_NORMED", + cv2.TM_CCORR: "TM_CCORR", + cv2.TM_CCORR_NORMED: "TM_CCORR_NORMED", + cv2.TM_SQDIFF: "TM_SQDIFF", + cv2.TM_SQDIFF_NORMED: "TM_SQDIFF_NORMED", + } + return names.get(method, str(method)) diff --git a/find_python.bat b/find_python.bat index 489650d..686953e 100644 --- a/find_python.bat +++ b/find_python.bat @@ -6,6 +6,8 @@ if defined PYTHON goto :_find_python_done for %%C in ( + "%LOCALAPPDATA%\miniforge3\envs\botty\python.exe" + "%LOCALAPPDATA%\miniconda3\envs\botty\python.exe" "%USERPROFILE%\miniforge3\envs\botty\python.exe" "%USERPROFILE%\miniconda3\envs\botty\python.exe" "%USERPROFILE%\anaconda3\envs\botty\python.exe" diff --git a/fixtures/ocr_samples/.gitkeep b/fixtures/ocr_samples/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/fixtures/ocr_samples/.gitkeep @@ -0,0 +1 @@ + diff --git a/fixtures/screenshots/sample_scene.ppm b/fixtures/screenshots/sample_scene.ppm new file mode 100644 index 0000000..f852f38 --- /dev/null +++ b/fixtures/screenshots/sample_scene.ppm @@ -0,0 +1,11 @@ +P3 +8 8 +255 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 0 0 0 0 0 0 +0 0 0 10 10 10 10 10 10 255 0 0 0 255 0 0 0 255 0 0 0 0 0 0 +0 0 0 10 10 10 10 10 10 0 255 0 0 0 255 255 0 0 0 0 0 0 0 0 +0 0 0 10 10 10 10 10 10 0 0 255 255 255 0 0 255 255 0 0 0 0 0 0 +0 0 0 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 diff --git a/fixtures/templates/sample_marker.ppm b/fixtures/templates/sample_marker.ppm new file mode 100644 index 0000000..b5dde2b --- /dev/null +++ b/fixtures/templates/sample_marker.ppm @@ -0,0 +1,6 @@ +P3 +3 3 +255 +255 0 0 0 255 0 0 0 255 +0 255 0 0 0 255 255 0 0 +0 0 255 255 255 0 0 255 255 diff --git a/install.bat b/install.bat index de62fca..af9ca9e 100644 --- a/install.bat +++ b/install.bat @@ -153,6 +153,8 @@ echo. :: --- Find botty Python --- set "PYTHON=" for %%C in ( + "%LOCALAPPDATA%\miniforge3\envs\botty\python.exe" + "%LOCALAPPDATA%\miniconda3\envs\botty\python.exe" "%USERPROFILE%\miniforge3\envs\botty\python.exe" "%USERPROFILE%\miniconda3\envs\botty\python.exe" "%USERPROFILE%\anaconda3\envs\botty\python.exe" @@ -180,6 +182,8 @@ if %errorlevel% neq 0 ( :: Re-scan for Python after env creation set "PYTHON=" for %%C in ( + "%LOCALAPPDATA%\miniforge3\envs\botty\python.exe" + "%LOCALAPPDATA%\miniconda3\envs\botty\python.exe" "%USERPROFILE%\miniforge3\envs\botty\python.exe" "%USERPROFILE%\miniconda3\envs\botty\python.exe" "%USERPROFILE%\anaconda3\envs\botty\python.exe" @@ -231,6 +235,8 @@ echo Setting up OCR... :: Find the botty env directory set "BOTTY_ENV_DIR=" for %%C in ( + "%LOCALAPPDATA%\miniforge3\envs\botty" + "%LOCALAPPDATA%\miniconda3\envs\botty" "%USERPROFILE%\miniforge3\envs\botty" "%USERPROFILE%\miniconda3\envs\botty" "%USERPROFILE%\anaconda3\envs\botty" @@ -284,6 +290,11 @@ if exist "dependencies\tesserocr.cp310-win_amd64.pyd" ( :: --- Backend 2: pytesseract (reliable fallback) --- :: Needs tesseract.exe from winget. Works on any Python version. :: On Win10: winget may not be available -- if install fails, offer manual link. +"%CONDA_EXE%" run -n botty python -m pip install --progress-bar off pytesseract >nul 2>&1 +if %errorlevel% neq 0 ( + echo WARNING: Could not install pytesseract Python wrapper. +) + if not exist "C:\Program Files\Tesseract-OCR\tesseract.exe" ( echo Installing Tesseract OCR via winget... winget install --id tesseract-ocr.tesseract --silent --accept-package-agreements --accept-source-agreements 2>nul @@ -331,7 +342,7 @@ if "%OCR_READY%"=="0" ( echo. echo Verifying all dependencies... set "ALL_OK=1" -for %%M in (cv2 mss numpy transitions rapidfuzz) do ( +for %%M in (cv2 mss numpy transitions rapidfuzz pydantic pytesseract yaml) do ( set "PATH=%TESS_PATH%;%PATH%" "%CONDA_EXE%" run -n botty python -c "import %%M" >nul 2>&1 if !errorlevel! neq 0 ( diff --git a/pyproject.toml b/pyproject.toml index cc1a386..9a9bc46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,8 @@ dependencies = [ "transitions", "mss==7.0.1", "numpy==1.26.4", + "pydantic", + "PyYAML", "beautifultable", "pytweening", "requests", @@ -26,6 +28,9 @@ dependencies = [ # tesserocr must be installed separately - see development.md ] +[project.scripts] +botty-next = "botty_next.cli:main" + [project.optional-dependencies] dev = [ "pytest", diff --git a/requirements.txt b/requirements.txt index e2afd77..77a2b80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,59 +1,43 @@ -# ========================================== -# botty - Pixelbot for Diablo 2 Resurrected -# requirements.txt -# ========================================== -# Pin to versions that match the existing conda environment (environment.yml). -# Run `pip install -r requirements.txt` inside the `botty` conda env. -# For tesserocr, install the pre-built wheel instead: -# pip install dependencies/tesserocr-2.5.2-cp310-cp310-win_amd64.whl -# ========================================== - -# --- Core runtime --- +aiohappyeyeballs==2.6.2 +aiohttp==3.14.1 +aiosignal==1.4.0 +async-timeout==5.1.0 +attrs==26.1.0 +beautifultable==1.1.0 +certifi==2026.6.17 +cffi==2.0.0 +charset-normalizer==3.4.7 +colorama==0.4.6 +cryptography==49.0.0 +dataclasses-json==0.6.7 +discord.py==2.7.1 +frozenlist==1.5.0 +idna==3.10 +keyboard==0.13.5 +marshmallow==3.26.2 +mouse==0.7.1 +mss==7.0.1 +multidict==6.7.1 numpy==1.26.4 opencv-python==4.5.5.64 -mss==7.0.1 -pillow -pywin32 - -# --- Game input & control --- -keyboard -mouse -pytweening - -# --- State machine & logic --- -transitions -rapidfuzz==2.15.1 -pyparsing -parse -dataclasses-json - -# --- UI & display --- -colorama -beautifultable - -# --- Networking & messaging --- -requests -discord.py - -# --- System & utilities --- -psutil -cryptography -typing_extensions -graphviz - -# --- OCR --- -# pytesseract: pure-Python fallback; calls the system tesseract.exe installed -# by install.bat (winget). Always available regardless of tesserocr DLL state. -pytesseract -# tesserocr (fast path): install via bundled wheel AFTER conda tesseract 4.x: -# pip install dependencies/tesserocr-2.5.2-cp310-cp310-win_amd64.whl - -# --- Dev / testing --- -pytest -pytest-env -pytest-pythonpath -pytest-mock -coverage - -# --- Packaging --- -pyinstaller +packaging==25.0 +parse==1.22.1 +pillow==12.2.0 +psutil==7.2.2 +pycparser==2.22 +pyparsing==3.3.2 +pytesseract==0.3.13 +pytweening==1.2.0 +PyYAML==6.0.3 +rapidfuzz==3.12.1 +requests==2.34.2 +six==1.17.0 +transitions==0.9.3 +typing-inspect==0.9.0 +typing_extensions==4.15.0 +urllib3==2.7.0 +wcwidth==0.2.13 +python-dotenv==1.2.2 +graphviz==0.21 +Pygments==2.20.0 +pydantic==2.12.5 diff --git a/run_gems_all.bat b/run_gems_all.bat new file mode 100644 index 0000000..2e15260 --- /dev/null +++ b/run_gems_all.bat @@ -0,0 +1,31 @@ +@echo off +setlocal enabledelayedexpansion + +set "BOTTY_DIR=%~dp0" +cd /d "%BOTTY_DIR%" + +call "%BOTTY_DIR%find_python.bat" +if errorlevel 1 exit /b 1 + +for %%F in ("%PYTHON%") do set "_ENV=%%~dpF" +set "_ENV=%_ENV:~0,-1%" + +set "PATH=%_ENV%;%_ENV%\Scripts;%_ENV%\Library\bin;%_ENV%\Library\mingw-w64\bin;%_ENV%\Library\usr\bin;%_ENV%\DLLs;%PATH%" +set "CONDA_PREFIX=%_ENV%" +set "PYTHONUTF8=1" +set "PYTHONIOENCODING=utf-8" +set "SSL_CERT_DIR=" +set "TESSDATA_PREFIX=%_ENV%\Library\share" +set "PYTESSERACT_TESSERACT_CMD=C:\Program Files\Tesseract-OCR\tesseract.exe" + +echo Running gem transmute testbed... +echo Usage examples: +echo run_gems_all.bat +echo run_gems_all.bat flawless +echo run_gems_all.bat flawless diamond --max 1 +echo. + +"%PYTHON%" "%BOTTY_DIR%tools\testbed.py" gems_all %* +set "RC=%ERRORLEVEL%" +pause +exit /b %RC% diff --git a/src/transmute/transmute.py b/src/transmute/transmute.py index 7632051..3be7fdc 100644 --- a/src/transmute/transmute.py +++ b/src/transmute/transmute.py @@ -5,7 +5,7 @@ import io import urllib.request from random import randint from config import Config -from ui_manager import detect_screen_object, select_screen_object_match, wait_until_visible, ScreenObjects +from ui_manager import detect_screen_object, is_visible, select_screen_object_match, wait_until_visible, ScreenObjects from .inventory_collection import InventoryCollection from .stash import Stash from .gem_picking import SimpleGemPicking @@ -71,6 +71,46 @@ PERFECT_GEMS = [ "INVENTORY_SKULL_PERFECT" ] +GEMS_TAB_STACK_COORDS = { + # GEMS tab stack grid, screen coordinates at 1280x720. + # Columns are fixed by gem family; rows are fixed by tier. + "INVENTORY_DIAMOND_CHIPPED": (85, 154), + "INVENTORY_EMERALD_CHIPPED": (132, 154), + "INVENTORY_RUBY_CHIPPED": (180, 154), + "INVENTORY_TOPAZ_CHIPPED": (227, 154), + "INVENTORY_AMETHYST_CHIPPED": (275, 154), + "INVENTORY_SAPPHIRE_CHIPPED": (322, 154), + "INVENTORY_SKULL_CHIPPED": (370, 154), + "INVENTORY_DIAMOND_FLAWED": (85, 195), + "INVENTORY_EMERALD_FLAWED": (132, 195), + "INVENTORY_RUBY_FLAWED": (180, 195), + "INVENTORY_TOPAZ_FLAWED": (227, 195), + "INVENTORY_AMETHYST_FLAWED": (275, 195), + "INVENTORY_SAPPHIRE_FLAWED": (322, 195), + "INVENTORY_SKULL_FLAWED": (370, 195), + "INVENTORY_DIAMOND_STANDARD": (85, 234), + "INVENTORY_EMERALD_STANDARD": (132, 234), + "INVENTORY_RUBY_STANDARD": (180, 234), + "INVENTORY_TOPAZ_STANDARD": (227, 234), + "INVENTORY_AMETHYST_STANDARD": (275, 234), + "INVENTORY_SAPPHIRE_STANDARD": (322, 234), + "INVENTORY_SKULL_STANDARD": (370, 234), + "INVENTORY_DIAMOND_FLAWLESS": (85, 273), + "INVENTORY_EMERALD_FLAWLESS": (132, 273), + "INVENTORY_RUBY_FLAWLESS": (180, 273), + "INVENTORY_TOPAZ_FLAWLESS": (227, 273), + "INVENTORY_AMETHYST_FLAWLESS": (275, 273), + "INVENTORY_SAPPHIRE_FLAWLESS": (322, 273), + "INVENTORY_SKULL_FLAWLESS": (370, 273), + "INVENTORY_DIAMOND_PERFECT": (85, 311), + "INVENTORY_EMERALD_PERFECT": (132, 311), + "INVENTORY_RUBY_PERFECT": (180, 311), + "INVENTORY_TOPAZ_PERFECT": (227, 311), + "INVENTORY_AMETHYST_PERFECT": (275, 311), + "INVENTORY_SAPPHIRE_PERFECT": (322, 311), + "INVENTORY_SKULL_PERFECT": (370, 311), +} + class Transmute: @staticmethod @@ -202,8 +242,11 @@ class Transmute: # Named-tab stash UI (1280x720). 5 tabs ~76px wide starting at x=33: # PERSONAL≈71 SHARED≈147 GEMS≈223 MATERIALS≈299 RUNES≈375 PERSONAL_TAB_X = 71 - GEMS_TAB_X = 200 + GEMS_TAB_X = 223 GEMS_TAB_Y = 100 + GEMS_CONVERT_BUTTON = (225, 500) + GEMS_CONVERT_PANEL_ROI = (160, 296, 128, 160) + GEMS_CONVERT_FIRST_SLOT = (181, 318) def _switch_to_personal_tab(self) -> None: x, y = convert_screen_to_monitor((self.PERSONAL_TAB_X, self.GEMS_TAB_Y)) @@ -219,42 +262,46 @@ class Transmute: mouse.click("left") wait(0.4, 0.5) + def _click_gems_tab_convert_button(self) -> None: + """Click the native convert button in the GEMS tab panel.""" + x, y = convert_screen_to_monitor(self.GEMS_CONVERT_BUTTON) + Logger.info(f" click GEMS convert button @ monitor ({x},{y})") + mouse.move(x, y) + self._wait() + mouse.click("left") + self._wait() + def _open_cube_from_stash(self) -> bool: - """Switch to PERSONAL tab and right-click the Horadric Cube to open its UI.""" - self._switch_to_personal_tab() + """Switch to GEMS tab and right-click the Horadric Cube to open its UI.""" + self._switch_to_gems_tab() if (match := detect_screen_object(ScreenObjects.CubeStash)).valid: mouse.move(*match.center_monitor) self._wait() mouse.click("right") wait(0.3, 0.4) return True - Logger.error("Cube not found in PERSONAL stash tab") + Logger.error("Cube not found in GEMS stash tab") return False def _open_cube_from_gems_tab(self) -> bool: - """Open the Horadric Cube from PERSONAL stash, then return to GEMS tab.""" - if not self._open_cube_from_stash(): - return False - self._switch_to_gems_tab() - wait(0.3, 0.4) - return True + """Open the Horadric Cube from the GEMS stash tab.""" + return self._open_cube_from_stash() def _open_available_cube_for_gems_tab(self) -> bool: - """Open the cube from the PERSONAL stash, then return to GEMS tab.""" + """Open or confirm the cube UI while the GEMS tab is active.""" if detect_screen_object(ScreenObjects.CubeOpened).valid: self._switch_to_gems_tab() wait(0.3, 0.4) return True - self._switch_to_personal_tab() + self._switch_to_gems_tab() + if detect_screen_object(ScreenObjects.CubeOpened).valid: + return True if detect_screen_object(ScreenObjects.CubeStash).valid: return self._open_cube_from_gems_tab() if detect_screen_object(ScreenObjects.CubeInventory).valid: - Logger.error( - "_open_available_cube_for_gems_tab: cube is in character inventory; " - "move it to the PERSONAL stash tab before running gems_all" - ) - return False - Logger.error("_open_available_cube_for_gems_tab: cube not found in PERSONAL stash") + Logger.info("_open_available_cube_for_gems_tab: opening cube from character inventory with GEMS active") + return self._open_cube_from_inventory() + Logger.error("_open_available_cube_for_gems_tab: cube not found on GEMS stash tab") return False @@ -279,32 +326,40 @@ class Transmute: self._wait() def _ctrl_shift_click_monitor(self, x: int, y: int, label: str = "") -> None: - """Ctrl+Shift+Right-Click — moves item from current stash tab into the Horadric Cube.""" - from input_layer.win_input import key_down, key_up, _get_vk + """Ctrl+Shift+Right-Click helper retained for older inventory flows.""" Logger.info(f" ctrl+shift+rclick {label} @ monitor ({x},{y})") mouse.move(x, y) self._wait() - key_down(_get_vk('ctrl')) - key_down(_get_vk('shift')) - self._wait() - mouse.click("right") - key_up(_get_vk('shift')) - key_up(_get_vk('ctrl')) - self._wait() + try: + keyboard.send("ctrl", do_release=False) + keyboard.send("shift", do_release=False) + self._wait() + mouse.click("right") + finally: + keyboard.release("shift") + keyboard.release("ctrl") + self._wait() def _ctrl_shift_left_click_monitor(self, x: int, y: int, label: str = "") -> None: - """Ctrl+Shift+Left-Click — moves item from Horadric Cube back to current stash tab.""" - from input_layer.win_input import key_down, key_up, _get_vk + """Ctrl+Shift+Left-Click for GEMS-tab moves between stack slots and the convert panel.""" Logger.info(f" ctrl+shift+lclick {label} @ monitor ({x},{y})") mouse.move(x, y) self._wait() - key_down(_get_vk('ctrl')) - key_down(_get_vk('shift')) - self._wait() - mouse.click("left") - key_up(_get_vk('shift')) - key_up(_get_vk('ctrl')) - self._wait() + try: + keyboard.send("ctrl", do_release=False) + keyboard.send("shift", do_release=False) + self._wait() + mouse.click("left") + finally: + keyboard.release("shift") + keyboard.release("ctrl") + self._wait() + + def _gems_stack_monitor_for(self, gem_template: str) -> tuple[int, int] | None: + pos = GEMS_TAB_STACK_COORDS.get(gem_template) + if pos is None: + return None + return convert_screen_to_monitor(pos) def _gems_tab_pull_3(self, gems_in: list) -> int: """Find a gem type in the GEMS tab (stacked) and ctrl+click it 3 times to pull 3 gems.""" @@ -336,7 +391,9 @@ class Transmute: def _locate_cube(self) -> str | None: """Return 'stash' or 'inventory' depending on where the cube is, or None if not found.""" - self._switch_to_personal_tab() + self._switch_to_gems_tab() + if detect_screen_object(ScreenObjects.CubeOpened).valid: + return 'opened' if detect_screen_object(ScreenObjects.CubeStash).valid: return 'stash' if detect_screen_object(ScreenObjects.CubeInventory).valid: @@ -346,53 +403,39 @@ class Transmute: def _ensure_cube_in_stash(self) -> bool: """Move the Horadric Cube to the stash PERSONAL tab if it's currently in character inventory. - Only works if the cube is in the loot columns (0 to num_loot_columns-1); cols 4+ are - restricted by the equipped-area safety guard and require manual relocation to the stash.""" + This only clicks a positive CubeInventory match, so it is safe to move from reserved + inventory columns as well as loot columns.""" self._switch_to_personal_tab() if detect_screen_object(ScreenObjects.CubeStash).valid: Logger.info(" cube already in stash PERSONAL tab") return True match = detect_screen_object(ScreenObjects.CubeInventory) if match.valid: - from screen import convert_monitor_to_screen - screen_x = convert_monitor_to_screen(match.center_monitor)[0] - loot_cols = Config().char["num_loot_columns"] - slot_w = Config().ui_pos["slot_width"] - inv_x0 = Config().ui_roi["right_inventory"][0] - loot_end_x = inv_x0 + loot_cols * slot_w - if screen_x < loot_end_x: - Logger.info(" moving cube from inventory loot cols to stash PERSONAL tab") - self._ctrl_click_monitor(*match.center_monitor, label="cube") - wait(0.5, 0.6) - if detect_screen_object(ScreenObjects.CubeStash).valid: - return True - Logger.error("_ensure_cube_in_stash: ctrl+click did not land cube in stash") - return False - else: - Logger.error( - "_ensure_cube_in_stash: Horadric Cube is in restricted inventory cols " - f"(screen_x={screen_x}, loot ends at {loot_end_x}). " - "Move the Cube to the stash PERSONAL tab manually before running gems_all." - ) - return False + Logger.info(" moving cube from character inventory to stash PERSONAL tab") + self._ctrl_click_monitor(*match.center_monitor, label="cube") + wait(0.5, 0.6) + if detect_screen_object(ScreenObjects.CubeStash).valid: + return True + Logger.error("_ensure_cube_in_stash: ctrl+click did not land cube in stash") + return False Logger.error("_ensure_cube_in_stash: cube not found in stash or character inventory") return False def _ensure_cube_available(self) -> bool: - """Verify the cube is in PERSONAL stash for the GEMS-tab flow.""" + """Verify the cube can be used while the GEMS tab is active.""" if detect_screen_object(ScreenObjects.CubeOpened).valid: Logger.info(" cube UI already open") return True location = self._locate_cube() + if location == "opened": + Logger.info(" cube UI available on GEMS tab") + return True if location == "stash": - Logger.info(" cube available in stash") + Logger.info(" cube available on GEMS stash tab") return True if location == "inventory": - Logger.error( - " cube is in character inventory; move it to the PERSONAL stash tab " - "before running gems_all" - ) - return False + Logger.info(" cube available in character inventory; it will stay there") + return True return False def _reopen_stash(self) -> bool: @@ -584,24 +627,31 @@ class Transmute: def _count_gems_by_ocr(self) -> dict: - """Count gem stacks in the GEMS tab using template matching + OCR of the count badge. + """Count gem stacks in the GEMS tab using fixed stack slots + OCR of the count badge. Each stacked slot has a small digit in the bottom-right corner of the icon. Returns dict keyed like 'inventory_ruby_flawless': N (or 999 if OCR failed).""" import re as _re from d2r_image import ocr as d2r_ocr img = grab() - roi = Config().ui_roi["left_inventory"] slot_w = Config().ui_pos["slot_width"] slot_h = Config().ui_pos["slot_height"] img_h, img_w = img.shape[:2] all_gems = CHIPPED_GEMS + FLAWED_GEMS + STANDARD_GEMS + FLAWLESS_GEMS + PERFECT_GEMS counts = {} for gem in all_gems: - matches = template_finder.search_all(gem, img, threshold=0.60, roi=roi) - if not matches: + center = GEMS_TAB_STACK_COORDS.get(gem) + if center is None: + continue + cx, cy = center + icon = img[ + max(0, cy - slot_h // 2):min(img_h, cy + slot_h // 2), + max(0, cx - slot_w // 2):min(img_w, cx + slot_w // 2), + ] + if icon.size == 0: + continue + gray_icon = cv2.cvtColor(icon, cv2.COLOR_BGR2GRAY) + if float(np.percentile(gray_icon, 95)) < 45: continue - best = max(matches, key=lambda m: m.score) - cx, cy = int(best.center[0]), int(best.center[1]) # Count badge sits in the bottom-right quadrant of the slot. x1 = max(0, cx) y1 = max(0, cy) @@ -740,13 +790,13 @@ class Transmute: return slots def _empty_cube_to_gems_tab(self) -> int: - """ctrl+shift+left-click all 12 cube slots → any items land on the current stash tab. + """ctrl+left-click all 12 cube slots → any items land on the active GEMS tab. Returns number of slots that had something (any item that moved makes a sound but we can't detect that here — we just blindly click all 12 and let D2R sort it out).""" slots = self._cube_slot_monitors() moved = 0 for mx, my in slots: - self._ctrl_shift_left_click_monitor(mx, my, label="cube-slot") + self._ctrl_click_monitor(mx, my, label="cube-slot") wait(0.12, 0.15) moved += 1 return moved @@ -761,14 +811,14 @@ class Transmute: Pre-conditions: - Stash must be open - - Horadric Cube must be in the PERSONAL stash tab + - GEMS tab must be available Per-transmute flow: - 1. Open the cube from PERSONAL stash - 2. Switch to GEMS while cube stays open - 3. Ctrl+shift+right-click gem x3 -> 3 gems load into cube - 4. Click Transmute - 5. Ctrl+shift+left-click result from cube -> result lands on GEMS tab + 1. Switch to GEMS + 2. Ctrl+Shift+Left-click gem x3 -> 3 gems load into the GEMS convert panel + 3. Click the GEMS tab convert button + 4. Ctrl+Shift+Left-click the result back to the GEMS tab + 5. Leave the character-inventory cube untouched tier_filter: optional list of tier names to process (e.g. ["chipped"]). None means all tiers. @@ -782,28 +832,6 @@ class Transmute: Logger.error("convert_all_gems: stash not open") return - if not self._ensure_cube_available(): - Logger.error("convert_all_gems: cube must be available in PERSONAL stash") - return - - # Open cube before clicking cube slots; otherwise these coordinates hit the stash panel. - if self._open_available_cube_for_gems_tab(): - self._switch_to_gems_tab() - wait(0.4, 0.5) - Logger.info("convert_all_gems: pre-flight clear (ctrl+shift+left-click 12 cube slots)") - self._empty_cube_to_gems_tab() - self.close_cube() - wait(0.4, 0.5) - else: - Logger.warning("convert_all_gems: skipping pre-flight cube clear because cube UI could not be opened") - - from ui_manager import is_visible - img = grab() - if not (is_visible(ScreenObjects.GoldBtnStash, img) or left_inventory_ready(img)): - if not self._reopen_stash(): - Logger.error("convert_all_gems: could not reopen stash after pre-flight cube clear") - return - # Switch to GEMS tab before counting and planning. self._switch_to_gems_tab() wait(0.4, 0.5) @@ -841,7 +869,6 @@ class Transmute: for i in range(n_transmutes): # Ensure stash is open and on GEMS tab - from ui_manager import is_visible from inventory.common import left_inventory_ready img = grab() if not (is_visible(ScreenObjects.GoldBtnStash, img) or left_inventory_ready(img)): @@ -852,66 +879,46 @@ class Transmute: self._switch_to_gems_tab() wait(0.3, 0.4) - img = grab() - matches = template_finder.search_all(gem_template, img, threshold=0.85, roi=left_roi) - if not matches: - Logger.info(f" no {gem_template} left in GEMS tab -- stopping {tier_name}") - break + pos = self._gems_stack_monitor_for(gem_template) + if pos is None: + img = grab() + matches = template_finder.search_all(gem_template, img, threshold=0.85, roi=left_roi) + if not matches: + Logger.info(f" no {gem_template} left in GEMS tab -- stopping {tier_name}") + break + pos = matches[0].center_monitor - # Step 1: open the cube, then return to GEMS. - if not self._open_available_cube_for_gems_tab(): - Logger.error(f" [{i+1}/{n_transmutes}] cube not found -- aborting") - return - wait(0.3, 0.4) - - # Step 2: ctrl+shift+right-click x3 -- 3 gems move from GEMS tab into the open cube - pos = matches[0].center_monitor - Logger.info(f" [{i+1}/{n_transmutes}] loading 3x {gem_template} into cube @ {pos}") + # Step 1: ctrl+shift+left-click x3 -- 3 gems move from GEMS tab into the convert panel. + Logger.info(f" [{i+1}/{n_transmutes}] loading 3x {gem_template} into GEMS convert panel @ {pos}") + loaded = 0 for _ in range(3): - self._ctrl_shift_click_monitor(*pos, label=gem_template) + self._ctrl_shift_left_click_monitor(*pos, label=gem_template) + loaded += 1 wait(0.25, 0.3) + if loaded != 3: + Logger.error(f" [{i+1}/{n_transmutes}] expected to load exactly 3 gems, loaded {loaded}; aborting") + return - # Step 3: transmute button visible in cube UI - wait(0.3, 0.4) - t_match = wait_until_visible(ScreenObjects.CubeOpened, timeout=3.0) - if not t_match.valid: - Logger.warning(f" [{i+1}/{n_transmutes}] transmute btn not visible -- clearing cube and stopping this gem type") - self._empty_cube_to_gems_tab() - self.close_cube() - wait(0.4, 0.5) - break - - # Step 4: click transmute - Logger.info(f" [{i+1}/{n_transmutes}] transmuting") - select_screen_object_match(t_match) + # Step 2: click the GEMS tab's own convert button. + Logger.info(f" [{i+1}/{n_transmutes}] converting via GEMS tab button") + self._click_gems_tab_convert_button() wait(0.6, 0.7) - # Step 5: move result directly from cube to the active GEMS tab. - cube_roi = Config().ui_roi["cube_area_roi"] + convert_roi = self.GEMS_CONVERT_PANEL_ROI img = grab() - result_match = template_finder.search(gems_out, img, threshold=0.72, roi=cube_roi) + result_match = template_finder.search(gems_out, img, threshold=0.72, roi=convert_roi) if result_match.valid: - Logger.info(f" [{i+1}/{n_transmutes}] ctrl+shift+lclick {result_match.name} from cube to GEMS @ {result_match.center_monitor}") + Logger.info(f" [{i+1}/{n_transmutes}] ctrl+shift+lclick {result_match.name} from convert panel to GEMS @ {result_match.center_monitor}") self._ctrl_shift_left_click_monitor(*result_match.center_monitor, label=result_match.name) else: + x, y = convert_screen_to_monitor(self.GEMS_CONVERT_FIRST_SLOT) Logger.warning( - f" [{i+1}/{n_transmutes}] expected result {gems_out} not found in cube -- " - "clearing cube and stopping this gem type" + f" [{i+1}/{n_transmutes}] expected result {gems_out} not found in GEMS convert panel; " + f"trying first slot fallback @ monitor ({x},{y})" ) - self._empty_cube_to_gems_tab() - self.close_cube() - wait(0.4, 0.5) - break + self._ctrl_shift_left_click_monitor(x, y, label="GEMS convert result fallback") wait(0.3, 0.4) - # Step 6: close cube, keep/reopen stash on GEMS tab - self.close_cube() - wait(0.4, 0.5) - img = grab() - if not (is_visible(ScreenObjects.GoldBtnStash, img) or left_inventory_ready(img)): - if not self._reopen_stash(): - Logger.error(f" [{i+1}/{n_transmutes}] could not reopen stash after cube close -- aborting") - return self._switch_to_gems_tab() wait(0.3, 0.4) diff --git a/test/transmute/transmute_test.py b/test/transmute/transmute_test.py index f098c05..e592290 100644 --- a/test/transmute/transmute_test.py +++ b/test/transmute/transmute_test.py @@ -72,20 +72,30 @@ class TestTransmutePlanning: assert len(diamonds) == 1 assert diamonds[0][1] == "INVENTORY_DIAMOND_CHIPPED" - def test_cube_available_rejects_inventory_cube(self, monkeypatch): + def test_gems_stack_coordinates_cover_all_transmutable_inputs(self): + all_inputs = ( + transmute_module.CHIPPED_GEMS + + transmute_module.FLAWED_GEMS + + transmute_module.STANDARD_GEMS + + transmute_module.FLAWLESS_GEMS + ) + + missing = [gem for gem in all_inputs if gem not in transmute_module.GEMS_TAB_STACK_COORDS] + + assert missing == [] + + def test_cube_available_accepts_inventory_cube_for_gems_tab_flow(self, monkeypatch): subject = object.__new__(Transmute) monkeypatch.setattr(subject, "_locate_cube", lambda: "inventory") - assert subject._ensure_cube_available() is False + assert subject._ensure_cube_available() is True - def test_open_available_cube_rejects_inventory_when_stash_missing(self, monkeypatch): + def test_open_available_cube_selects_gems_before_opening_inventory_cube(self, monkeypatch): subject = object.__new__(Transmute) calls = [] def detect(screen_object): calls.append(screen_object) - if screen_object is transmute_module.ScreenObjects.CubeStash: - return _Match(False) if screen_object is transmute_module.ScreenObjects.CubeInventory: return _Match(True) return _Match(False) @@ -93,12 +103,34 @@ class TestTransmutePlanning: monkeypatch.setattr(transmute_module, "detect_screen_object", detect) monkeypatch.setattr(subject, "_switch_to_personal_tab", lambda: calls.append("personal")) monkeypatch.setattr(subject, "_switch_to_gems_tab", lambda: calls.append("gems")) + monkeypatch.setattr(subject, "_open_cube_from_gems_tab", lambda: calls.append("open_stash_cube") or True) monkeypatch.setattr(subject, "_open_cube_from_inventory", lambda: calls.append("open_inventory") or True) monkeypatch.setattr(transmute_module, "wait", lambda *args, **kwargs: None) - assert subject._open_available_cube_for_gems_tab() is False - assert "open_inventory" not in calls - assert "gems" not in calls + assert subject._open_available_cube_for_gems_tab() is True + assert "gems" in calls + assert "personal" not in calls + assert "open_stash_cube" not in calls + assert "open_inventory" in calls + + def test_open_available_cube_opens_when_cube_is_on_gems_tab(self, monkeypatch): + subject = object.__new__(Transmute) + calls = [] + + def detect(screen_object): + calls.append(screen_object) + if screen_object is transmute_module.ScreenObjects.CubeStash: + return _Match(True) + return _Match(False) + + monkeypatch.setattr(transmute_module, "detect_screen_object", detect) + monkeypatch.setattr(subject, "_switch_to_gems_tab", lambda: calls.append("gems")) + monkeypatch.setattr(subject, "_open_cube_from_gems_tab", lambda: calls.append("open_stash_cube") or True) + monkeypatch.setattr(transmute_module, "wait", lambda *args, **kwargs: None) + + assert subject._open_available_cube_for_gems_tab() is True + assert "gems" in calls + assert "open_stash_cube" in calls def test_open_available_cube_accepts_already_open_cube_ui(self, monkeypatch): subject = object.__new__(Transmute) @@ -120,3 +152,57 @@ class TestTransmutePlanning: assert "gems" in calls assert "personal" not in calls assert "open_inventory" not in calls + + def test_gems_tab_convert_flow_does_not_touch_inventory_cube(self, monkeypatch): + subject = object.__new__(Transmute) + calls = [] + searched = {"count": 0} + + monkeypatch.setattr(transmute_module, "wait_until_visible", lambda *args, **kwargs: _Match(True)) + monkeypatch.setattr(transmute_module, "wait", lambda *args, **kwargs: None) + monkeypatch.setattr(subject, "_switch_to_gems_tab", lambda: calls.append("gems")) + monkeypatch.setattr(subject, "_count_gems_for_plan", lambda: {"diamond_chipped": 3}) + monkeypatch.setattr(subject, "_ctrl_shift_click_monitor", lambda *args, **kwargs: calls.append("wrong_load")) + monkeypatch.setattr(subject, "_ctrl_click_monitor", lambda *args, **kwargs: calls.append("wrong_return_result")) + def _ctrl_shift_left_click(*_args, **kwargs): + label = kwargs.get("label", "") + calls.append("return_result" if "FLAWED" in label else "load") + + monkeypatch.setattr(subject, "_ctrl_shift_left_click_monitor", _ctrl_shift_left_click) + monkeypatch.setattr(subject, "_click_gems_tab_convert_button", lambda: calls.append("convert")) + monkeypatch.setattr(subject, "_open_available_cube_for_gems_tab", lambda: calls.append("open_cube") or True) + monkeypatch.setattr(subject, "_ensure_cube_available", lambda: calls.append("ensure_cube") or True) + monkeypatch.setattr(subject, "_empty_cube_to_gems_tab", lambda: calls.append("empty_cube")) + monkeypatch.setattr(subject, "close_cube", lambda: calls.append("close_cube")) + monkeypatch.setattr(transmute_module, "is_visible", lambda *args, **kwargs: True) + + monkeypatch.setattr( + transmute_module, + "grab", + lambda: object(), + ) + + class _TemplateFinder: + @staticmethod + def search_all(*_args, **_kwargs): + searched["count"] += 1 + return [type("Match", (), {"center_monitor": (100, 100)})()] + + @staticmethod + def search(*_args, **_kwargs): + return type("Match", (), {"valid": True, "name": "INVENTORY_DIAMOND_FLAWED", "center_monitor": (120, 120)})() + + monkeypatch.setattr(transmute_module, "template_finder", _TemplateFinder) + + subject.convert_all_gems_to_perfect(tier_filter=["chipped"], gem_filter=["diamond"], max_transmutes=1) + + assert calls.count("load") == 3 + assert "convert" in calls + assert "return_result" in calls + assert searched["count"] == 0 + assert "wrong_load" not in calls + assert "wrong_return_result" not in calls + assert "open_cube" not in calls + assert "ensure_cube" not in calls + assert "empty_cube" not in calls + assert "close_cube" not in calls