alex/my-botty
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

Update FG price tracker, game stats, and add d2jsp collection tooling

Browse Source
This commit is contained in:
alex
2026-05-27 01:08:45 +02:00
parent 19f4b8a553
commit e3d6605595
10 changed files with 440 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/fg_daily_estimates.json
+69
View File
@@ -0,0 +1,69 @@
{
"generated_at": "2026-05-26T11:03:35.048241+00:00",
"mode": "offline",
"offline_dir": "data\\d2jsp_pages",
"ladder_start_date": "2026-05-20",
"days": 14,
"topics_scanned": 21,
"estimates": {
"day_1": {},
"day_2": {},
"day_3": {},
"day_4": {},
"day_5": {},
"day_6": {},
"day_7": {
"CTA": {
"median_fg": 1199.0,
"samples": 1
},
"Topic:Buy 2 Chaos Gc For 100 Fg": {
"median_fg": 100.0,
"samples": 2
},
"Cham Rune": {
"median_fg": 30.0,
"samples": 5
},
"Topic:Demon Skills Amulet And Skillers": {
"median_fg": 125.0,
"samples": 3
},
"Aldur's Advance": {
"median_fg": 25.0,
"samples": 3
},
"Ist Rune": {
"median_fg": 100.0,
"samples": 8
},
"Gul Rune": {
"median_fg": 65.0,
"samples": 4
},
"Topic:Mf Goldwrap, Nagel, Chancies": {
"median_fg": 35.0,
"samples": 3
},
"Dual Leech Ring": {
"median_fg": 70.0,
"samples": 1
},
"Unid Anni": {
"median_fg": 575.0,
"samples": 2
},
"Stealth RW": {
"median_fg": 10.0,
"samples": 1
}
},
"day_8": {},
"day_9": {},
"day_10": {},
"day_11": {},
"day_12": {},
"day_13": {},
"day_14": {}
}
}
config/fg_prices.json
+22 -1
View File
@@ -2,6 +2,25 @@
"enabled": true,
"notes": "Day 3 ladder snapshot pricing. Values are per item in fg.",
"rules": [
{ "name": "El Rune", "match": "exact", "pattern": "EL RUNE", "fg": 3 },
{ "name": "Eld Rune", "match": "exact", "pattern": "ELD RUNE", "fg": 3 },
{ "name": "Tir Rune", "match": "exact", "pattern": "TIR RUNE", "fg": 3 },
{ "name": "Nef Rune", "match": "exact", "pattern": "NEF RUNE", "fg": 3 },
{ "name": "Eth Rune", "match": "exact", "pattern": "ETH RUNE", "fg": 3 },
{ "name": "Ith Rune", "match": "exact", "pattern": "ITH RUNE", "fg": 3 },
{ "name": "Tal Rune", "match": "exact", "pattern": "TAL RUNE", "fg": 5 },
{ "name": "Ral Rune", "match": "exact", "pattern": "RAL RUNE", "fg": 6 },
{ "name": "Ort Rune", "match": "exact", "pattern": "ORT RUNE", "fg": 8 },
{ "name": "Amn Rune", "match": "exact", "pattern": "AMN RUNE", "fg": 10 },
{ "name": "Sol Rune", "match": "exact", "pattern": "SOL RUNE", "fg": 20 },
{ "name": "Shael Rune", "match": "exact", "pattern": "SHAEL RUNE", "fg": 10 },
{ "name": "Dol Rune", "match": "exact", "pattern": "DOL RUNE", "fg": 10 },
{ "name": "Hel Rune", "match": "exact", "pattern": "HEL RUNE", "fg": 10 },
{ "name": "Io Rune", "match": "exact", "pattern": "IO RUNE", "fg": 12 },
{ "name": "Ko Rune", "match": "exact", "pattern": "KO RUNE", "fg": 12 },
{ "name": "Lum Rune", "match": "exact", "pattern": "LUM RUNE", "fg": 12 },
{ "name": "Fal Rune", "match": "exact", "pattern": "FAL RUNE", "fg": 20 },
{ "name": "Cham Rune", "match": "exact", "pattern": "CHAM RUNE", "fg": 800 },
{ "name": "Lo Rune", "match": "exact", "pattern": "LO RUNE", "fg": 900 },
{ "name": "Ohm Rune", "match": "exact", "pattern": "OHM RUNE", "fg": 600 },
@@ -55,6 +74,8 @@
{ "name": "Pcomb SK", "match": "contains", "pattern": "PALADIN COMBAT", "fg": 200 },
{ "name": "Cold SK", "match": "contains", "pattern": "COLD SKILLS", "fg": 250 },
{ "name": "Java SK", "match": "contains", "pattern": "JAVELIN", "fg": 200 },
{ "name": "Light SK", "match": "contains", "pattern": "LIGHTNING SKILLS", "fg": 200 }
{ "name": "Light SK", "match": "contains", "pattern": "LIGHTNING SKILLS", "fg": 200 },
{ "name": "Eth War Scythe 4os (Insight base)", "match": "contains", "pattern": "ETH WAR SCYTHE", "fg": 25 }
]
}
src/fg_price_tracker.py
+7 -1
View File
@@ -15,6 +15,10 @@ class FGPriceTracker:
def _normalize(text: str) -> str:
return " ".join((text or "").strip().upper().split())
@staticmethod
def _compact(text: str) -> str:
return "".join(ch for ch in (text or "").upper() if ch.isalnum())
def _load(self):
cfg_candidates = [self._cfg_path, os.path.join("..", self._cfg_path)]
cfg_path = next((p for p in cfg_candidates if os.path.exists(p)), None)
@@ -45,11 +49,13 @@ class FGPriceTracker:
if not self._enabled or not self._rules:
return None, None
normalized = self._normalize(item_name)
compact = self._compact(item_name)
for rule in self._rules:
if rule["match"] == "contains":
if rule["pattern"] in normalized:
return rule["fg"], rule["name"] or rule["pattern"]
else:
if normalized == rule["pattern"]:
# Be tolerant to OCR/style variants like "ELRUNE" vs "EL RUNE".
if normalized == rule["pattern"] or compact == self._compact(rule["pattern"]):
return rule["fg"], rule["name"] or rule["pattern"]
return None, None
src/game_stats.py
+7 -2
View File
@@ -5,6 +5,7 @@ import threading
import inspect
import json
import os
import re
from beautifultable import BeautifulTable
from logger import Logger
@@ -138,7 +139,11 @@ class GameStats:
@staticmethod
def _is_rune(item_name: str) -> bool:
return GameStats._normalize_item_name(item_name).endswith(" RUNE")
normalized = GameStats._normalize_item_name(item_name)
if normalized.endswith(" RUNE"):
return True
compact = re.sub(r"[^A-Z0-9]", "", normalized)
return compact.endswith("RUNE")
def log_item_keep(self, item_name: str, send_message: bool, img: np.ndarray, ocr_text: str = '', expression: str = '', item_props: dict = {}):
filtered_substrings = [" POTION", " OF IDENTIFY", " OF TOWN PORTAL", " AMETHYST", " RUBY", " TOPAZ", " EMERALD", " SAPPHIRE", " DIAMOND"]
@@ -384,7 +389,7 @@ class GameStats:
self._location_stats["totals"]["failed_runs"]
])
table.columns.header = ["Run", "I", "C", "D", "MD", "F"]
table.columns.header = ["Run", "I", "C", "D", "MD", "FR"]
msg += f"\n{str(table)}\n"
return msg
tools/apply_manual_cookies.py
+15
View File
@@ -0,0 +1,15 @@
def main():
# Netscape format: domain \t TRUE/FALSE \t path \t TRUE/FALSE \t expiry \t name \t value
# Expiry 2000000000 is roughly the year 2033.
cookies = [
(".d2jsp.org\tTRUE\t/\tFALSE\t2000000000\tmember_id\tREDACTED_MEMBER_ID"),
(".d2jsp.org\tTRUE\t/\tFALSE\t2000000000\tmsec\tREDACTED_D2JSP_MSEC"),
]
with open("cookies.txt", "w") as f:
f.write("# Netscape HTTP Cookie File\n")
f.write("\n".join(cookies) + "\n")
print("Successfully created cookies.txt with manual session data.")
if __name__ == "__main__":
main()
tools/browser_auto_save.py
+47
View File
@@ -0,0 +1,47 @@
import webbrowser
import time
import keyboard
import os
from pathlib import Path
def main():
url_file = Path("data/d2jsp_topic_urls.txt")
if not url_file.exists():
print("Error: data/d2jsp_topic_urls.txt not found.")
return
urls = url_file.read_text().splitlines()
urls = [u.strip() for u in urls if u.strip()][:20] # Limit to 20 as requested
print(f"Starting auto-save for {len(urls)} topics.")
print("!!! IMPORTANT !!!")
print("1. Ensure your browser is the active window.")
print("2. Save ONE page manually to the 'data/d2jsp_pages' folder first to set the default path.")
print("3. Do not touch your keyboard/mouse until finished.")
print("Starting in 5 seconds...")
time.sleep(5)
for i, url in enumerate(urls, 1):
print(f"[{i}/{len(urls)}] Opening: {url}")
webbrowser.open_new_tab(url)
# Wait for page to load and Cloudflare to pass
time.sleep(8)
# Press Ctrl+S
keyboard.press_and_release('ctrl+s')
time.sleep(2)
# Press Enter to confirm save
keyboard.press_and_release('enter')
time.sleep(3)
# Press Ctrl+W to close tab
keyboard.press_and_release('ctrl+w')
time.sleep(1)
print("\nFinished! Now you can run the scraper:")
print("python tools/fg_market_scraper.py --ladder-start-date 2026-05-20 --offline-dir data/d2jsp_pages --out config/fg_daily_estimates.json")
if __name__ == "__main__":
main()
tools/diag_cookies.py
+45
View File
@@ -0,0 +1,45 @@
import sqlite3
import shutil
import os
from pathlib import Path
def main():
paths = [
("Chrome", Path(os.environ["USERPROFILE"]) / "AppData" / "Local" / "Google" / "Chrome" / "User Data" / "Default" / "Network" / "Cookies"),
("Edge", Path(os.environ["USERPROFILE"]) / "AppData" / "Local" / "Microsoft" / "Edge" / "User Data" / "Default" / "Network" / "Cookies"),
]
for name, db_path in paths:
if not db_path.exists():
print(f"{name} Cookies DB not found.")
continue
print(f"\n--- {name} ---")
temp_db = f"diag_cookies_{name}.db"
try:
shutil.copy(db_path, temp_db)
except Exception as e:
print(f"Failed to copy: {e}")
continue
try:
conn = sqlite3.connect(temp_db)
cursor = conn.cursor()
cursor.execute("SELECT host_key, name, encrypted_value FROM cookies WHERE host_key LIKE '%d2jsp.org%'")
rows = cursor.fetchall()
if rows:
print(f"Found {len(rows)} d2jsp cookies. Inspecting first few:")
for host, name, val in rows[:5]:
prefix = val[:3]
print(f" {host} | {name} | prefix={prefix}")
else:
print("No d2jsp cookies found.")
conn.close()
except Exception as e:
print(f"Error: {e}")
finally:
if os.path.exists(temp_db):
os.remove(temp_db)
if __name__ == "__main__":
main()
tools/get_cookies.py
+46
View File
@@ -0,0 +1,46 @@
import browser_cookie3
from http.cookiejar import MozillaCookieJar
import sys
from pathlib import Path
def save_cookies(cj, filename):
mj = MozillaCookieJar(filename)
for cookie in cj:
mj.set_cookie(cookie)
mj.save(ignore_discard=True, ignore_expires=True)
def main():
domain = 'forums.d2jsp.org'
found = False
browsers = [
('Chrome', browser_cookie3.chrome),
('Edge', browser_cookie3.edge),
('Firefox', browser_cookie3.firefox),
('Opera', browser_cookie3.opera),
('Brave', browser_cookie3.brave),
]
print(f"Searching for {domain} cookies...")
for name, func in browsers:
try:
print(f"Trying {name}...", end=' ', flush=True)
cj = func(domain_name=domain)
count = len(list(cj))
print(f"found {count} cookies.")
if count > 0:
save_cookies(cj, 'cookies.txt')
print(f"Successfully saved {count} cookies from {name} to cookies.txt")
found = True
break
except Exception as e:
print(f"failed: {e}")
if not found:
print("\nCould not find any d2jsp cookies automatically.")
print("Please ensure you are logged in to forums.d2jsp.org in one of your browsers.")
sys.exit(1)
if __name__ == "__main__":
main()
tools/get_cookies_manual.py
+139
View File
@@ -0,0 +1,139 @@
import os
import json
import base64
import sqlite3
import shutil
from pathlib import Path
import win32crypt
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
def get_master_key(user_data_path):
local_state_path = user_data_path / "Local State"
if not local_state_path.exists():
return None
with open(local_state_path, "r", encoding="utf-8") as f:
local_state = json.load(f)
encrypted_key = base64.b64decode(local_state["os_crypt"]["encrypted_key"])
# Remove DPAPI prefix
encrypted_key = encrypted_key[5:]
master_key = win32crypt.CryptUnprotectData(encrypted_key, None, None, None, 0)[1]
return master_key
def generate_netscape_cookie(domain, name, value, path, expiry):
# Netscape cookie format:
# domain \t TRUE/FALSE \t path \t TRUE/FALSE \t expiry \t name \t value
return f"{domain}\tTRUE\t{path}\tFALSE\t{expiry}\t{name}\t{value}\n"
def get_firefox_cookies():
ff_path = Path(os.environ["APPDATA"]) / "Mozilla" / "Firefox" / "Profiles"
if not ff_path.exists():
return []
cookies = []
for profile in ff_path.glob("*"):
db_path = profile / "cookies.sqlite"
if not db_path.exists():
continue
print(f"Searching Firefox profile: {profile.name}...")
temp_db = f"cookies_temp_ff_{profile.name}.sqlite"
try:
shutil.copy(db_path, temp_db)
except:
continue
try:
conn = sqlite3.connect(temp_db)
cursor = conn.cursor()
cursor.execute("SELECT host, name, path, expiry, value FROM moz_cookies WHERE host LIKE '%d2jsp.org%'")
for host, name, path, expiry, value in cursor.fetchall():
cookies.append(generate_netscape_cookie(host, name, value, path, expiry))
conn.close()
except Exception as e:
print(f" Error reading Firefox database: {e}")
finally:
if os.path.exists(temp_db):
os.remove(temp_db)
return cookies
def main():
browsers = [
("Chrome", Path(os.environ["USERPROFILE"]) / "AppData" / "Local" / "Google" / "Chrome" / "User Data"),
("Edge", Path(os.environ["USERPROFILE"]) / "AppData" / "Local" / "Microsoft" / "Edge" / "User Data"),
]
cookies_txt = []
# Try Chromium-based
for browser_name, user_data_path in browsers:
if not user_data_path.exists():
continue
print(f"Searching {browser_name} cookies...")
master_key = get_master_key(user_data_path)
if not master_key:
print(f" {browser_name} master key not found.")
continue
aes_gcm = AESGCM(master_key)
profile_dirs = ["Default"] + [p.name for p in user_data_path.glob("Profile *")]
for profile in profile_dirs:
db_path = user_data_path / profile / "Network" / "Cookies"
if not db_path.exists():
db_path = user_data_path / profile / "Cookies"
if not db_path.exists():
continue
print(f" Checking profile: {profile}...")
temp_db = f"cookies_temp_{browser_name}_{profile}.db"
try:
shutil.copy(db_path, temp_db)
except Exception as e:
print(f" Access denied to {db_path}. Browser is likely open.")
continue
try:
conn = sqlite3.connect(temp_db)
cursor = conn.cursor()
cursor.execute("SELECT host_key, name, path, expires_utc, encrypted_value FROM cookies WHERE host_key LIKE '%d2jsp.org%'")
for host, name, path, expires, encrypted_value in cursor.fetchall():
try:
if encrypted_value.startswith(b'v10'):
decrypted_value = aes_gcm.decrypt(encrypted_value[3:15], encrypted_value[15:], None).decode('utf-8')
else:
decrypted_value = win32crypt.CryptUnprotectData(encrypted_value, None, None, None, 0)[1].decode('utf-8')
expiry_seconds = (expires // 1000000) - 11644473600 if expires > 0 else 0
cookies_txt.append(generate_netscape_cookie(host, name, decrypted_value, path, int(max(0, expiry_seconds))))
except:
pass
conn.close()
except Exception as e:
print(f" Error reading database: {e}")
finally:
if os.path.exists(temp_db):
os.remove(temp_db)
# Try Firefox
cookies_txt.extend(get_firefox_cookies())
if cookies_txt:
unique_cookies = {}
for line in cookies_txt:
parts = line.split('\t')
unique_cookies[(parts[0], parts[5])] = line
with open("cookies.txt", "w") as f:
f.write("# Netscape HTTP Cookie File\n")
f.writelines(unique_cookies.values())
print(f"Successfully saved {len(unique_cookies)} cookies to cookies.txt")
else:
print("\nNo d2jsp cookies found.")
print("ACTION REQUIRED: Please CLOSE your browser (Chrome/Edge/Firefox) and run this again, OR log in to d2jsp in your browser.")
if __name__ == "__main__":
main()
tools/test_access.py
+43
View File
@@ -0,0 +1,43 @@
import requests
import time
from http.cookiejar import MozillaCookieJar
def main():
session = requests.Session()
cookiejar = MozillaCookieJar("cookies.txt")
cookiejar.load(ignore_discard=True, ignore_expires=True)
session.cookies = cookiejar
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Language": "en-DK,en;q=0.9,da-DK;q=0.8,da;q=0.7,en-GB;q=0.6,en-US;q=0.5",
"Referer": "https://www.google.com/",
"DNT": "1",
"Upgrade-Insecure-Requests": "1",
"sec-ch-ua": '"Chromium";v="148", "Google Chrome";v="148", "Not/A)Brand";v="99"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"Windows"',
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "cross-site",
"sec-fetch-user": "?1",
}
print("Step 1: Hitting root...")
r1 = session.get("https://forums.d2jsp.org/", headers=headers, timeout=20)
print(f"Status: {r1.status_code}, Length: {len(r1.text)}")
if r1.status_code == 200:
print("Step 2: Hitting forum...")
r2 = session.get("https://forums.d2jsp.org/forum.php?f=271", headers=headers, timeout=20)
print(f"Status: {r2.status_code}, Length: {len(r2.text)}")
if "Just a moment..." in r2.text:
print("Blocked by Cloudflare challenge.")
else:
print("Access granted!")
else:
print("Root hit failed.")
if __name__ == "__main__":
main()