diff --git a/environment.yml b/environment.yml index f3090fb..145e07e 100644 --- a/environment.yml +++ b/environment.yml @@ -31,3 +31,4 @@ dependencies: - parse - dependencies/tesserocr-2.5.2-cp39-cp39-win_amd64.whl - typing_extensions + - rapidfuzz diff --git a/src/inventory/common.py b/src/inventory/common.py index 1f8a369..5bb6793 100644 --- a/src/inventory/common.py +++ b/src/inventory/common.py @@ -87,6 +87,8 @@ def calc_item_roi(img_pre, img_post): final = np.bitwise_and.reduce([blue_red_mask, green_mask, diff_thresh]) _, roi = trim_black(final) return roi + except ValueError: + Logger.debug(f"_calc_item_roi: Couldn't determine item dimensions--tooltip likely obscuring") except BaseException as err: Logger.error(f"_calc_item_roi: Unexpected {err=}, {type(err)=}") return None diff --git a/src/item/item_cropper.py b/src/item/item_cropper.py index 4a357ce..6bba97d 100644 --- a/src/item/item_cropper.py +++ b/src/item/item_cropper.py @@ -140,10 +140,11 @@ class ItemCropper: if __name__ == "__main__": import keyboard import os - from screen import grab, start_detecting_window - from template_finder import TemplateFinder - + from screen import start_detecting_window, grab start_detecting_window() + keyboard.add_hotkey('f12', lambda: Logger.info('Force Exit (f12)') or os._exit(1)) + print("Move to d2r window and press f11") + keyboard.wait("f11") keyboard.add_hotkey('f12', lambda: os._exit(1)) cropper = ItemCropper() @@ -151,9 +152,23 @@ if __name__ == "__main__": while 1: img = grab().copy() res = cropper.crop_item_descr(img, model="engd2r_inv_th_fast") - if res["color"]: + if res.valid: x, y, w, h = res.roi cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), 1) - Logger.debug(f"{res.ocr_result['text']}") + #Logger.debug(f"{res.ocr_result['text']}") + + Logger.debug(f"OCR ITEM DESCR: Mean conf: {res.ocr_result.mean_confidence}") + for i, line in enumerate(list(filter(None, res.ocr_result.text.splitlines()))): + Logger.debug(f"OCR LINE{i}: {line}") + timestamp = time.strftime("%Y%m%d_%H%M%S") + found_low_confidence = False + for cnt, x in enumerate(res.ocr_result['word_confidences']): + if x <= 88: + try: + Logger.debug(f"Low confidence word #{cnt}: {res.ocr_result['original_text'].split()[cnt]} -> {res.ocr_result['text'].split()[cnt]}, Conf: {x}") + found_low_confidence = True + except: pass + + cv2.imshow("res", img) - cv2.waitKey(1) + cv2.waitKey(5000) diff --git a/src/ocr.py b/src/ocr.py index 269570b..24a176e 100644 --- a/src/ocr.py +++ b/src/ocr.py @@ -2,8 +2,9 @@ from tesserocr import PyTessBaseAPI, PSM, OEM import numpy as np import cv2 import re +from rapidfuzz.process import extractOne +from rapidfuzz.string_metric import levenshtein import csv -import difflib from utils.misc import erode_to_black from logger import Logger from typing import List, Union @@ -77,7 +78,7 @@ class Ocr: fix_regexps: bool = True, check_known_errors: bool = True, check_wordlist: bool = True, - word_match_threshold: float = 0.9 + word_match_threshold: float = 0.5 ) -> list[str]: """ Uses Tesseract to read image(s) @@ -156,13 +157,14 @@ class Ocr: """ OCR output processing functions: """ + def _check_known_errors(self, text): for key, value in self._ocr_errors.items(): if key in text: text = text.replace(key, value) return text - def _check_wordlist(self, text: str = None, word_list: str = None, confidences: list = [], match_threshold: float = 0.9) -> str: + def _check_wordlist(self, text: str = None, word_list: str = None, confidences: list = [], match_threshold: float = 0.5) -> str: with open(f'assets/tessdata/word_lists/{word_list}') as file: word_list = [line.rstrip() for line in file] @@ -173,12 +175,14 @@ class Ocr: word = word.strip() if word and word != "NEWLINEHERE": try: - if confidences[word_count] <= 88: - if (word not in word_list) and (re.sub(r"[^a-zA-Z0-9]", "", word) not in word_list): - closest_match = difflib.get_close_matches(word, word_list, cutoff=match_threshold) - if closest_match and closest_match != word: - new_string += f"{closest_match[0]} " - Logger.debug(f"check_wordlist: Replacing {word} ({confidences[word_count]}%) with {closest_match[0]}, score=") + if confidences[word_count] <= 90: + alphanumeric = re.sub(r"[^a-zA-Z0-9]", "", word) + if not alphanumeric.isnumeric() and (word not in word_list) and alphanumeric not in word_list: + closest_match, similarity, _ = extractOne(word, word_list, scorer=levenshtein) + normalized_similarity = 1 - similarity / len(word) + if (normalized_similarity) >= (match_threshold): + new_string += f"{closest_match} " + Logger.debug(f"check_wordlist: Replacing {word} ({confidences[word_count]}%) with {closest_match}, similarity={normalized_similarity*100:.1f}%") else: new_string += f"{word} " else: @@ -190,8 +194,8 @@ class Ocr: # bizarre word_count index exceeded sometimes... can't reproduce and words otherwise seem to match up Logger.error(f"check_wordlist: IndexError for word: {word}, index: {word_count}, text: {text}") return text - except: - Logger.error(f"check_wordlist: Unknown error for word: {word}, index: {word_count}, text: {text}") + except Exception as e: + Logger.error(f"check_wordlist: Unknown error for word: {word}, index: {word_count}, text: {text}, exception: {e}") return text elif word == "NEWLINEHERE": new_string += "\n" @@ -300,6 +304,6 @@ if __name__ == "__main__": fix_regexps = False, check_known_errors = False, check_wordlist = False, - word_match_threshold = 0.9 + word_match_threshold = 0.5 )[0] Logger.debug(ocr_result.text) \ No newline at end of file diff --git a/src/ui/player_bar.py b/src/ui/player_bar.py index 037c011..3703a55 100644 --- a/src/ui/player_bar.py +++ b/src/ui/player_bar.py @@ -27,7 +27,7 @@ def get_experience(): fix_regexps = False, check_known_errors = False, check_wordlist = False, - word_match_threshold = 0.9 + word_match_threshold = 0.5 )[0] split_text = ocr_result.text.split(' ') current_exp = int(split_text[1].replace(',', '')) diff --git a/src/ui/skills.py b/src/ui/skills.py index dca8983..60dc4ad 100644 --- a/src/ui/skills.py +++ b/src/ui/skills.py @@ -87,7 +87,7 @@ def get_skill_charges(ocr, img: np.ndarray = None): fix_regexps = False, check_known_errors = False, check_wordlist = False, - word_match_threshold = 0.9 + word_match_threshold = 0.5 )[0] try: return int(ocr_result.text)