diff --git a/config/fg_daily_estimates.json b/config/fg_daily_estimates.json new file mode 100644 index 0000000..2fda1fa --- /dev/null +++ b/config/fg_daily_estimates.json @@ -0,0 +1,69 @@ +{ + "generated_at": "2026-05-26T11:03:35.048241+00:00", + "mode": "offline", + "offline_dir": "data\\d2jsp_pages", + "ladder_start_date": "2026-05-20", + "days": 14, + "topics_scanned": 21, + "estimates": { + "day_1": {}, + "day_2": {}, + "day_3": {}, + "day_4": {}, + "day_5": {}, + "day_6": {}, + "day_7": { + "CTA": { + "median_fg": 1199.0, + "samples": 1 + }, + "Topic:Buy 2 Chaos Gc For 100 Fg": { + "median_fg": 100.0, + "samples": 2 + }, + "Cham Rune": { + "median_fg": 30.0, + "samples": 5 + }, + "Topic:Demon Skills Amulet And Skillers": { + "median_fg": 125.0, + "samples": 3 + }, + "Aldur's Advance": { + "median_fg": 25.0, + "samples": 3 + }, + "Ist Rune": { + "median_fg": 100.0, + "samples": 8 + }, + "Gul Rune": { + "median_fg": 65.0, + "samples": 4 + }, + "Topic:Mf Goldwrap, Nagel, Chancies": { + "median_fg": 35.0, + "samples": 3 + }, + "Dual Leech Ring": { + "median_fg": 70.0, + "samples": 1 + }, + "Unid Anni": { + "median_fg": 575.0, + "samples": 2 + }, + "Stealth RW": { + "median_fg": 10.0, + "samples": 1 + } + }, + "day_8": {}, + "day_9": {}, + "day_10": {}, + "day_11": {}, + "day_12": {}, + "day_13": {}, + "day_14": {} + } +} \ No newline at end of file diff --git a/config/fg_prices.json b/config/fg_prices.json index 7aec71e..9be9528 100644 --- a/config/fg_prices.json +++ b/config/fg_prices.json @@ -2,6 +2,25 @@ "enabled": true, "notes": "Day 3 ladder snapshot pricing. Values are per item in fg.", "rules": [ + { "name": "El Rune", "match": "exact", "pattern": "EL RUNE", "fg": 3 }, + { "name": "Eld Rune", "match": "exact", "pattern": "ELD RUNE", "fg": 3 }, + { "name": "Tir Rune", "match": "exact", "pattern": "TIR RUNE", "fg": 3 }, + { "name": "Nef Rune", "match": "exact", "pattern": "NEF RUNE", "fg": 3 }, + { "name": "Eth Rune", "match": "exact", "pattern": "ETH RUNE", "fg": 3 }, + { "name": "Ith Rune", "match": "exact", "pattern": "ITH RUNE", "fg": 3 }, + { "name": "Tal Rune", "match": "exact", "pattern": "TAL RUNE", "fg": 5 }, + { "name": "Ral Rune", "match": "exact", "pattern": "RAL RUNE", "fg": 6 }, + { "name": "Ort Rune", "match": "exact", "pattern": "ORT RUNE", "fg": 8 }, + { "name": "Amn Rune", "match": "exact", "pattern": "AMN RUNE", "fg": 10 }, + { "name": "Sol Rune", "match": "exact", "pattern": "SOL RUNE", "fg": 20 }, + { "name": "Shael Rune", "match": "exact", "pattern": "SHAEL RUNE", "fg": 10 }, + { "name": "Dol Rune", "match": "exact", "pattern": "DOL RUNE", "fg": 10 }, + { "name": "Hel Rune", "match": "exact", "pattern": "HEL RUNE", "fg": 10 }, + { "name": "Io Rune", "match": "exact", "pattern": "IO RUNE", "fg": 12 }, + { "name": "Ko Rune", "match": "exact", "pattern": "KO RUNE", "fg": 12 }, + { "name": "Lum Rune", "match": "exact", "pattern": "LUM RUNE", "fg": 12 }, + { "name": "Fal Rune", "match": "exact", "pattern": "FAL RUNE", "fg": 20 }, + { "name": "Cham Rune", "match": "exact", "pattern": "CHAM RUNE", "fg": 800 }, { "name": "Lo Rune", "match": "exact", "pattern": "LO RUNE", "fg": 900 }, { "name": "Ohm Rune", "match": "exact", "pattern": "OHM RUNE", "fg": 600 }, @@ -55,6 +74,8 @@ { "name": "Pcomb SK", "match": "contains", "pattern": "PALADIN COMBAT", "fg": 200 }, { "name": "Cold SK", "match": "contains", "pattern": "COLD SKILLS", "fg": 250 }, { "name": "Java SK", "match": "contains", "pattern": "JAVELIN", "fg": 200 }, - { "name": "Light SK", "match": "contains", "pattern": "LIGHTNING SKILLS", "fg": 200 } + { "name": "Light SK", "match": "contains", "pattern": "LIGHTNING SKILLS", "fg": 200 }, + + { "name": "Eth War Scythe 4os (Insight base)", "match": "contains", "pattern": "ETH WAR SCYTHE", "fg": 25 } ] } diff --git a/src/fg_price_tracker.py b/src/fg_price_tracker.py index c38b0a9..0fb9127 100644 --- a/src/fg_price_tracker.py +++ b/src/fg_price_tracker.py @@ -15,6 +15,10 @@ class FGPriceTracker: def _normalize(text: str) -> str: return " ".join((text or "").strip().upper().split()) + @staticmethod + def _compact(text: str) -> str: + return "".join(ch for ch in (text or "").upper() if ch.isalnum()) + def _load(self): cfg_candidates = [self._cfg_path, os.path.join("..", self._cfg_path)] cfg_path = next((p for p in cfg_candidates if os.path.exists(p)), None) @@ -45,11 +49,13 @@ class FGPriceTracker: if not self._enabled or not self._rules: return None, None normalized = self._normalize(item_name) + compact = self._compact(item_name) for rule in self._rules: if rule["match"] == "contains": if rule["pattern"] in normalized: return rule["fg"], rule["name"] or rule["pattern"] else: - if normalized == rule["pattern"]: + # Be tolerant to OCR/style variants like "ELRUNE" vs "EL RUNE". + if normalized == rule["pattern"] or compact == self._compact(rule["pattern"]): return rule["fg"], rule["name"] or rule["pattern"] return None, None diff --git a/src/game_stats.py b/src/game_stats.py index f9fe1dc..a6834cd 100644 --- a/src/game_stats.py +++ b/src/game_stats.py @@ -5,6 +5,7 @@ import threading import inspect import json import os +import re from beautifultable import BeautifulTable from logger import Logger @@ -138,7 +139,11 @@ class GameStats: @staticmethod def _is_rune(item_name: str) -> bool: - return GameStats._normalize_item_name(item_name).endswith(" RUNE") + normalized = GameStats._normalize_item_name(item_name) + if normalized.endswith(" RUNE"): + return True + compact = re.sub(r"[^A-Z0-9]", "", normalized) + return compact.endswith("RUNE") def log_item_keep(self, item_name: str, send_message: bool, img: np.ndarray, ocr_text: str = '', expression: str = '', item_props: dict = {}): filtered_substrings = [" POTION", " OF IDENTIFY", " OF TOWN PORTAL", " AMETHYST", " RUBY", " TOPAZ", " EMERALD", " SAPPHIRE", " DIAMOND"] @@ -384,7 +389,7 @@ class GameStats: self._location_stats["totals"]["failed_runs"] ]) - table.columns.header = ["Run", "I", "C", "D", "MD", "F"] + table.columns.header = ["Run", "I", "C", "D", "MD", "FR"] msg += f"\n{str(table)}\n" return msg diff --git a/tools/apply_manual_cookies.py b/tools/apply_manual_cookies.py new file mode 100644 index 0000000..c7b3fee --- /dev/null +++ b/tools/apply_manual_cookies.py @@ -0,0 +1,15 @@ +def main(): + # Netscape format: domain \t TRUE/FALSE \t path \t TRUE/FALSE \t expiry \t name \t value + # Expiry 2000000000 is roughly the year 2033. + cookies = [ + (".d2jsp.org\tTRUE\t/\tFALSE\t2000000000\tmember_id\tREDACTED_MEMBER_ID"), + (".d2jsp.org\tTRUE\t/\tFALSE\t2000000000\tmsec\tREDACTED_D2JSP_MSEC"), + ] + + with open("cookies.txt", "w") as f: + f.write("# Netscape HTTP Cookie File\n") + f.write("\n".join(cookies) + "\n") + print("Successfully created cookies.txt with manual session data.") + +if __name__ == "__main__": + main() diff --git a/tools/browser_auto_save.py b/tools/browser_auto_save.py new file mode 100644 index 0000000..ace5d8f --- /dev/null +++ b/tools/browser_auto_save.py @@ -0,0 +1,47 @@ +import webbrowser +import time +import keyboard +import os +from pathlib import Path + +def main(): + url_file = Path("data/d2jsp_topic_urls.txt") + if not url_file.exists(): + print("Error: data/d2jsp_topic_urls.txt not found.") + return + + urls = url_file.read_text().splitlines() + urls = [u.strip() for u in urls if u.strip()][:20] # Limit to 20 as requested + + print(f"Starting auto-save for {len(urls)} topics.") + print("!!! IMPORTANT !!!") + print("1. Ensure your browser is the active window.") + print("2. Save ONE page manually to the 'data/d2jsp_pages' folder first to set the default path.") + print("3. Do not touch your keyboard/mouse until finished.") + print("Starting in 5 seconds...") + time.sleep(5) + + for i, url in enumerate(urls, 1): + print(f"[{i}/{len(urls)}] Opening: {url}") + webbrowser.open_new_tab(url) + + # Wait for page to load and Cloudflare to pass + time.sleep(8) + + # Press Ctrl+S + keyboard.press_and_release('ctrl+s') + time.sleep(2) + + # Press Enter to confirm save + keyboard.press_and_release('enter') + time.sleep(3) + + # Press Ctrl+W to close tab + keyboard.press_and_release('ctrl+w') + time.sleep(1) + + print("\nFinished! Now you can run the scraper:") + print("python tools/fg_market_scraper.py --ladder-start-date 2026-05-20 --offline-dir data/d2jsp_pages --out config/fg_daily_estimates.json") + +if __name__ == "__main__": + main() diff --git a/tools/diag_cookies.py b/tools/diag_cookies.py new file mode 100644 index 0000000..e0b6952 --- /dev/null +++ b/tools/diag_cookies.py @@ -0,0 +1,45 @@ +import sqlite3 +import shutil +import os +from pathlib import Path + +def main(): + paths = [ + ("Chrome", Path(os.environ["USERPROFILE"]) / "AppData" / "Local" / "Google" / "Chrome" / "User Data" / "Default" / "Network" / "Cookies"), + ("Edge", Path(os.environ["USERPROFILE"]) / "AppData" / "Local" / "Microsoft" / "Edge" / "User Data" / "Default" / "Network" / "Cookies"), + ] + + for name, db_path in paths: + if not db_path.exists(): + print(f"{name} Cookies DB not found.") + continue + + print(f"\n--- {name} ---") + temp_db = f"diag_cookies_{name}.db" + try: + shutil.copy(db_path, temp_db) + except Exception as e: + print(f"Failed to copy: {e}") + continue + + try: + conn = sqlite3.connect(temp_db) + cursor = conn.cursor() + cursor.execute("SELECT host_key, name, encrypted_value FROM cookies WHERE host_key LIKE '%d2jsp.org%'") + rows = cursor.fetchall() + if rows: + print(f"Found {len(rows)} d2jsp cookies. Inspecting first few:") + for host, name, val in rows[:5]: + prefix = val[:3] + print(f" {host} | {name} | prefix={prefix}") + else: + print("No d2jsp cookies found.") + conn.close() + except Exception as e: + print(f"Error: {e}") + finally: + if os.path.exists(temp_db): + os.remove(temp_db) + +if __name__ == "__main__": + main() diff --git a/tools/get_cookies.py b/tools/get_cookies.py new file mode 100644 index 0000000..00894eb --- /dev/null +++ b/tools/get_cookies.py @@ -0,0 +1,46 @@ +import browser_cookie3 +from http.cookiejar import MozillaCookieJar +import sys +from pathlib import Path + +def save_cookies(cj, filename): + mj = MozillaCookieJar(filename) + for cookie in cj: + mj.set_cookie(cookie) + mj.save(ignore_discard=True, ignore_expires=True) + +def main(): + domain = 'forums.d2jsp.org' + found = False + + browsers = [ + ('Chrome', browser_cookie3.chrome), + ('Edge', browser_cookie3.edge), + ('Firefox', browser_cookie3.firefox), + ('Opera', browser_cookie3.opera), + ('Brave', browser_cookie3.brave), + ] + + print(f"Searching for {domain} cookies...") + + for name, func in browsers: + try: + print(f"Trying {name}...", end=' ', flush=True) + cj = func(domain_name=domain) + count = len(list(cj)) + print(f"found {count} cookies.") + if count > 0: + save_cookies(cj, 'cookies.txt') + print(f"Successfully saved {count} cookies from {name} to cookies.txt") + found = True + break + except Exception as e: + print(f"failed: {e}") + + if not found: + print("\nCould not find any d2jsp cookies automatically.") + print("Please ensure you are logged in to forums.d2jsp.org in one of your browsers.") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/tools/get_cookies_manual.py b/tools/get_cookies_manual.py new file mode 100644 index 0000000..8908136 --- /dev/null +++ b/tools/get_cookies_manual.py @@ -0,0 +1,139 @@ +import os +import json +import base64 +import sqlite3 +import shutil +from pathlib import Path +import win32crypt +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +def get_master_key(user_data_path): + local_state_path = user_data_path / "Local State" + if not local_state_path.exists(): + return None + with open(local_state_path, "r", encoding="utf-8") as f: + local_state = json.load(f) + + encrypted_key = base64.b64decode(local_state["os_crypt"]["encrypted_key"]) + # Remove DPAPI prefix + encrypted_key = encrypted_key[5:] + master_key = win32crypt.CryptUnprotectData(encrypted_key, None, None, None, 0)[1] + return master_key + +def generate_netscape_cookie(domain, name, value, path, expiry): + # Netscape cookie format: + # domain \t TRUE/FALSE \t path \t TRUE/FALSE \t expiry \t name \t value + return f"{domain}\tTRUE\t{path}\tFALSE\t{expiry}\t{name}\t{value}\n" + +def get_firefox_cookies(): + ff_path = Path(os.environ["APPDATA"]) / "Mozilla" / "Firefox" / "Profiles" + if not ff_path.exists(): + return [] + + cookies = [] + for profile in ff_path.glob("*"): + db_path = profile / "cookies.sqlite" + if not db_path.exists(): + continue + + print(f"Searching Firefox profile: {profile.name}...") + temp_db = f"cookies_temp_ff_{profile.name}.sqlite" + try: + shutil.copy(db_path, temp_db) + except: + continue + + try: + conn = sqlite3.connect(temp_db) + cursor = conn.cursor() + cursor.execute("SELECT host, name, path, expiry, value FROM moz_cookies WHERE host LIKE '%d2jsp.org%'") + for host, name, path, expiry, value in cursor.fetchall(): + cookies.append(generate_netscape_cookie(host, name, value, path, expiry)) + conn.close() + except Exception as e: + print(f" Error reading Firefox database: {e}") + finally: + if os.path.exists(temp_db): + os.remove(temp_db) + return cookies + +def main(): + browsers = [ + ("Chrome", Path(os.environ["USERPROFILE"]) / "AppData" / "Local" / "Google" / "Chrome" / "User Data"), + ("Edge", Path(os.environ["USERPROFILE"]) / "AppData" / "Local" / "Microsoft" / "Edge" / "User Data"), + ] + + cookies_txt = [] + + # Try Chromium-based + for browser_name, user_data_path in browsers: + if not user_data_path.exists(): + continue + + print(f"Searching {browser_name} cookies...") + + master_key = get_master_key(user_data_path) + if not master_key: + print(f" {browser_name} master key not found.") + continue + + aes_gcm = AESGCM(master_key) + profile_dirs = ["Default"] + [p.name for p in user_data_path.glob("Profile *")] + + for profile in profile_dirs: + db_path = user_data_path / profile / "Network" / "Cookies" + if not db_path.exists(): + db_path = user_data_path / profile / "Cookies" + if not db_path.exists(): + continue + + print(f" Checking profile: {profile}...") + temp_db = f"cookies_temp_{browser_name}_{profile}.db" + try: + shutil.copy(db_path, temp_db) + except Exception as e: + print(f" Access denied to {db_path}. Browser is likely open.") + continue + + try: + conn = sqlite3.connect(temp_db) + cursor = conn.cursor() + cursor.execute("SELECT host_key, name, path, expires_utc, encrypted_value FROM cookies WHERE host_key LIKE '%d2jsp.org%'") + + for host, name, path, expires, encrypted_value in cursor.fetchall(): + try: + if encrypted_value.startswith(b'v10'): + decrypted_value = aes_gcm.decrypt(encrypted_value[3:15], encrypted_value[15:], None).decode('utf-8') + else: + decrypted_value = win32crypt.CryptUnprotectData(encrypted_value, None, None, None, 0)[1].decode('utf-8') + + expiry_seconds = (expires // 1000000) - 11644473600 if expires > 0 else 0 + cookies_txt.append(generate_netscape_cookie(host, name, decrypted_value, path, int(max(0, expiry_seconds)))) + except: + pass + conn.close() + except Exception as e: + print(f" Error reading database: {e}") + finally: + if os.path.exists(temp_db): + os.remove(temp_db) + + # Try Firefox + cookies_txt.extend(get_firefox_cookies()) + + if cookies_txt: + unique_cookies = {} + for line in cookies_txt: + parts = line.split('\t') + unique_cookies[(parts[0], parts[5])] = line + + with open("cookies.txt", "w") as f: + f.write("# Netscape HTTP Cookie File\n") + f.writelines(unique_cookies.values()) + print(f"Successfully saved {len(unique_cookies)} cookies to cookies.txt") + else: + print("\nNo d2jsp cookies found.") + print("ACTION REQUIRED: Please CLOSE your browser (Chrome/Edge/Firefox) and run this again, OR log in to d2jsp in your browser.") + +if __name__ == "__main__": + main() diff --git a/tools/test_access.py b/tools/test_access.py new file mode 100644 index 0000000..531cd34 --- /dev/null +++ b/tools/test_access.py @@ -0,0 +1,43 @@ +import requests +import time +from http.cookiejar import MozillaCookieJar + +def main(): + session = requests.Session() + cookiejar = MozillaCookieJar("cookies.txt") + cookiejar.load(ignore_discard=True, ignore_expires=True) + session.cookies = cookiejar + + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Accept-Language": "en-DK,en;q=0.9,da-DK;q=0.8,da;q=0.7,en-GB;q=0.6,en-US;q=0.5", + "Referer": "https://www.google.com/", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "sec-ch-ua": '"Chromium";v="148", "Google Chrome";v="148", "Not/A)Brand";v="99"', + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": '"Windows"', + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "cross-site", + "sec-fetch-user": "?1", + } + + print("Step 1: Hitting root...") + r1 = session.get("https://forums.d2jsp.org/", headers=headers, timeout=20) + print(f"Status: {r1.status_code}, Length: {len(r1.text)}") + + if r1.status_code == 200: + print("Step 2: Hitting forum...") + r2 = session.get("https://forums.d2jsp.org/forum.php?f=271", headers=headers, timeout=20) + print(f"Status: {r2.status_code}, Length: {len(r2.text)}") + if "Just a moment..." in r2.text: + print("Blocked by Cloudflare challenge.") + else: + print("Access granted!") + else: + print("Root hit failed.") + +if __name__ == "__main__": + main()