Bugfix/enhancement: Fix ocr output wordlist check, replace difflib with Lev (#658)

* init

* remove unused difflib import

* use built-in function

* small changes

environment.yml

@@ -31,3 +31,4 @@ dependencies:
     - parse
     - dependencies/tesserocr-2.5.2-cp39-cp39-win_amd64.whl
     - typing_extensions
+    - rapidfuzz

src/inventory/common.py

@@ -87,6 +87,8 @@ def calc_item_roi(img_pre, img_post):
         final = np.bitwise_and.reduce([blue_red_mask, green_mask, diff_thresh])
         _, roi = trim_black(final)
         return roi
+    except ValueError:
+        Logger.debug(f"_calc_item_roi: Couldn't determine item dimensions--tooltip likely obscuring")
     except BaseException as err:
         Logger.error(f"_calc_item_roi: Unexpected {err=}, {type(err)=}")
         return None

src/item/item_cropper.py

@@ -140,10 +140,11 @@ class ItemCropper:
 if __name__ == "__main__":
     import keyboard
     import os
-    from screen import grab, start_detecting_window
-    from template_finder import TemplateFinder
-
+    from screen import start_detecting_window, grab
     start_detecting_window()
+    keyboard.add_hotkey('f12', lambda: Logger.info('Force Exit (f12)') or os._exit(1))
+    print("Move to d2r window and press f11")
+    keyboard.wait("f11")
 
     keyboard.add_hotkey('f12', lambda: os._exit(1))
     cropper = ItemCropper()
@@ -151,9 +152,23 @@ if __name__ == "__main__":
     while 1:
         img = grab().copy()
         res = cropper.crop_item_descr(img, model="engd2r_inv_th_fast")
-        if res["color"]:
+        if res.valid:
             x, y, w, h = res.roi
             cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), 1)
-            Logger.debug(f"{res.ocr_result['text']}")
+            #Logger.debug(f"{res.ocr_result['text']}")
+
+            Logger.debug(f"OCR ITEM DESCR: Mean conf: {res.ocr_result.mean_confidence}")
+            for i, line in enumerate(list(filter(None, res.ocr_result.text.splitlines()))):
+                Logger.debug(f"OCR LINE{i}: {line}")
+            timestamp = time.strftime("%Y%m%d_%H%M%S")
+            found_low_confidence = False
+            for cnt, x in enumerate(res.ocr_result['word_confidences']):
+                if x <= 88:
+                    try:
+                        Logger.debug(f"Low confidence word #{cnt}: {res.ocr_result['original_text'].split()[cnt]} -> {res.ocr_result['text'].split()[cnt]}, Conf: {x}")
+                        found_low_confidence = True
+                    except: pass
+
+
         cv2.imshow("res", img)
-        cv2.waitKey(1)
+        cv2.waitKey(5000)

src/ocr.py

@@ -2,8 +2,9 @@ from tesserocr import PyTessBaseAPI, PSM, OEM
 import numpy as np
 import cv2
 import re
+from rapidfuzz.process import extractOne
+from rapidfuzz.string_metric import levenshtein
 import csv
-import difflib
 from utils.misc import erode_to_black
 from logger import Logger
 from typing import List, Union
@@ -77,7 +78,7 @@ class Ocr:
         fix_regexps: bool = True,
         check_known_errors: bool = True,
         check_wordlist: bool = True,
-        word_match_threshold: float = 0.9
+        word_match_threshold: float = 0.5
     ) -> list[str]:
         """
         Uses Tesseract to read image(s)
@@ -156,13 +157,14 @@ class Ocr:
     """
     OCR output processing functions:
     """
+
     def _check_known_errors(self, text):
         for key, value in self._ocr_errors.items():
             if key in text:
                 text = text.replace(key, value)
         return text
 
-    def _check_wordlist(self, text: str = None, word_list: str = None, confidences: list = [], match_threshold: float = 0.9) -> str:
+    def _check_wordlist(self, text: str = None, word_list: str = None, confidences: list = [], match_threshold: float = 0.5) -> str:
         with open(f'assets/tessdata/word_lists/{word_list}') as file:
             word_list = [line.rstrip() for line in file]
 
@@ -173,12 +175,14 @@ class Ocr:
             word = word.strip()
             if word and word != "NEWLINEHERE":
                 try:
-                    if confidences[word_count] <= 88:
-                        if (word not in word_list) and (re.sub(r"[^a-zA-Z0-9]", "", word) not in word_list):
-                            closest_match = difflib.get_close_matches(word, word_list, cutoff=match_threshold)
-                            if closest_match and closest_match != word:
-                                new_string += f"{closest_match[0]} "
-                                Logger.debug(f"check_wordlist: Replacing {word} ({confidences[word_count]}%) with {closest_match[0]}, score=")
+                    if confidences[word_count] <= 90:
+                        alphanumeric = re.sub(r"[^a-zA-Z0-9]", "", word)
+                        if not alphanumeric.isnumeric() and (word not in word_list) and alphanumeric not in word_list:
+                            closest_match, similarity, _ = extractOne(word, word_list, scorer=levenshtein)
+                            normalized_similarity = 1 - similarity / len(word)
+                            if (normalized_similarity) >= (match_threshold):
+                                new_string += f"{closest_match} "
+                                Logger.debug(f"check_wordlist: Replacing {word} ({confidences[word_count]}%) with {closest_match}, similarity={normalized_similarity*100:.1f}%")
                             else:
                                 new_string += f"{word} "
                         else:
@@ -190,8 +194,8 @@ class Ocr:
                     # bizarre word_count index exceeded sometimes... can't reproduce and words otherwise seem to match up
                     Logger.error(f"check_wordlist: IndexError for word: {word}, index: {word_count}, text: {text}")
                     return text
-                except:
-                    Logger.error(f"check_wordlist: Unknown error for word: {word}, index: {word_count}, text: {text}")
+                except Exception as e:
+                    Logger.error(f"check_wordlist: Unknown error for word: {word}, index: {word_count}, text: {text}, exception: {e}")
                     return text
             elif word == "NEWLINEHERE":
                 new_string += "\n"
@@ -300,6 +304,6 @@ if __name__ == "__main__":
         fix_regexps = False,
         check_known_errors = False,
         check_wordlist = False,
-        word_match_threshold = 0.9
+        word_match_threshold = 0.5
     )[0]
     Logger.debug(ocr_result.text)

src/ui/player_bar.py

@@ -27,7 +27,7 @@ def get_experience():
         fix_regexps = False,
         check_known_errors = False,
         check_wordlist = False,
-        word_match_threshold = 0.9
+        word_match_threshold = 0.5
     )[0]
     split_text = ocr_result.text.split(' ')
     current_exp = int(split_text[1].replace(',', ''))

src/ui/skills.py

@@ -87,7 +87,7 @@ def get_skill_charges(ocr, img: np.ndarray = None):
         fix_regexps = False,
         check_known_errors = False,
         check_wordlist = False,
-        word_match_threshold = 0.9
+        word_match_threshold = 0.5
     )[0]
     try:
         return int(ocr_result.text)