alex/my-botty
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fix(input): snap cursor to exact target before click (SetCursorPos can land 1-2px off)

Browse Source 
D2R UI hitboxes are tight; a 1px offset on the Join Game tab click was
enough to miss the button. Final SetCursorPos pass after the existing
verification check.
This commit is contained in:
alexpolo1
2026-08-25 00:26:44 +02:00
parent a014fc538e
commit c26f249268
10 changed files with 2136 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files

700
config/fg_daily_estimates.json
View File

@@ -1,10 +1,10 @@
{
"generated_at": "2026-06-17T07:19:33.572415+00:00",
"generated_at": "2026-08-23T07:49:18.987798+00:00",
"mode": "offline",
"offline_dir": "data\\d2jsp_pages",
"ladder_start_date": "2026-05-12",
"days": 60,
"topics_scanned": 853,
"topics_scanned": 868,
"estimates": {
"day_1": {},
"day_2": {},
@@ -44,6 +44,55 @@
"max_fg": 1199.0,
"samples": 5
},
"Pul Rune": {
"median_fg": 60.0,
"avg_fg": 72.1,
"min_fg": 5.0,
"max_fg": 190.0,
"samples": 7
},
"Mal Rune": {
"median_fg": 100.0,
"avg_fg": 112.0,
"min_fg": 1.0,
"max_fg": 400.0,
"samples": 13
},
"Aldur's Advance": {
"median_fg": 27.5,
"avg_fg": 43.8,
"min_fg": 20.0,
"max_fg": 100.0,
"samples": 4
},
"Sorc Torch": {
"median_fg": 100.0,
"avg_fg": 100.0,
"min_fg": 100.0,
"max_fg": 100.0,
"samples": 1
},
"Um Rune": {
"median_fg": 75.0,
"avg_fg": 99.6,
"min_fg": 1.0,
"max_fg": 400.0,
"samples": 10
},
"Stealth RW": {
"median_fg": 55.0,
"avg_fg": 55.0,
"min_fg": 10.0,
"max_fg": 100.0,
"samples": 2
},
"Dual Leech Ring": {
"median_fg": 85.0,
"avg_fg": 85.0,
"min_fg": 70.0,
"max_fg": 100.0,
"samples": 2
},
"Cham Rune": {
"median_fg": 30.0,
"avg_fg": 121.0,
@@ -58,40 +107,12 @@
"max_fg": 100.0,
"samples": 4
},
"Um Rune": {
"median_fg": 75.0,
"avg_fg": 99.6,
"min_fg": 1.0,
"max_fg": 400.0,
"samples": 10
},
"Lem Rune": {
"median_fg": 60.0,
"avg_fg": 72.1,
"min_fg": 5.0,
"max_fg": 190.0,
"samples": 7
},
"Sorc Torch": {
"median_fg": 100.0,
"avg_fg": 100.0,
"Unid Anni": {
"median_fg": 550.0,
"avg_fg": 416.7,
"min_fg": 100.0,
"max_fg": 100.0,
"samples": 1
},
"Aldur's Advance": {
"median_fg": 27.5,
"avg_fg": 43.8,
"min_fg": 20.0,
"max_fg": 100.0,
"samples": 4
},
"Stealth RW": {
"median_fg": 55.0,
"avg_fg": 55.0,
"min_fg": 10.0,
"max_fg": 100.0,
"samples": 2
"max_fg": 600.0,
"samples": 3
},
"Ist Rune": {
"median_fg": 95.0,
@@ -100,34 +121,6 @@
"max_fg": 400.0,
"samples": 12
},
"Mal Rune": {
"median_fg": 100.0,
"avg_fg": 112.0,
"min_fg": 1.0,
"max_fg": 400.0,
"samples": 13
},
"Pul Rune": {
"median_fg": 60.0,
"avg_fg": 72.1,
"min_fg": 5.0,
"max_fg": 190.0,
"samples": 7
},
"Unid Anni": {
"median_fg": 550.0,
"avg_fg": 416.7,
"min_fg": 100.0,
"max_fg": 600.0,
"samples": 3
},
"Dual Leech Ring": {
"median_fg": 85.0,
"avg_fg": 85.0,
"min_fg": 70.0,
"max_fg": 100.0,
"samples": 2
},
"Gul Rune": {
"median_fg": 25.0,
"avg_fg": 63.1,
@@ -135,12 +128,12 @@
"max_fg": 200.0,
"samples": 8
},
"Key": {
"median_fg": 20.0,
"avg_fg": 18.3,
"Lem Rune": {
"median_fg": 60.0,
"avg_fg": 72.1,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
"max_fg": 190.0,
"samples": 7
},
"Insight": {
"median_fg": 25.0,
@@ -149,7 +142,28 @@
"max_fg": 400.0,
"samples": 6
},
"Ohm Rune": {
"Key": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"Oculus": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"Ber Rune": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"Java SK": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
@@ -163,21 +177,7 @@
"max_fg": 30.0,
"samples": 3
},
"White": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"Shako": {
"median_fg": 25.0,
"avg_fg": 91.0,
"min_fg": 1.0,
"max_fg": 400.0,
"samples": 6
},
"Black": {
"Jah Rune": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
@@ -198,13 +198,6 @@
"max_fg": 30.0,
"samples": 3
},
"Java SK": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"BK Ring": {
"median_fg": 20.0,
"avg_fg": 18.3,
@@ -212,6 +205,20 @@
"max_fg": 30.0,
"samples": 3
},
"Ohm Rune": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"Shako": {
"median_fg": 25.0,
"avg_fg": 91.0,
"min_fg": 1.0,
"max_fg": 400.0,
"samples": 6
},
"Skin": {
"median_fg": 20.0,
"avg_fg": 18.3,
@@ -219,20 +226,6 @@
"max_fg": 30.0,
"samples": 3
},
"Oculus": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"Spirit": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"Bone": {
"median_fg": 20.0,
"avg_fg": 18.3,
@@ -240,7 +233,7 @@
"max_fg": 30.0,
"samples": 3
},
"Jah Rune": {
"White": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
@@ -254,7 +247,14 @@
"max_fg": 30.0,
"samples": 3
},
"Ber Rune": {
"Spirit": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
"max_fg": 30.0,
"samples": 3
},
"Black": {
"median_fg": 20.0,
"avg_fg": 18.3,
"min_fg": 5.0,
@@ -288,491 +288,7 @@
"day_34": {},
"day_35": {},
"day_36": {},
"day_37": {
"Thresher": {
"median_fg": 32.5,
"avg_fg": 133.3,
"min_fg": 0.0,
"max_fg": 3500.0,
"samples": 210
},
"Monarch": {
"median_fg": 20.0,
"avg_fg": 71.5,
"min_fg": 0.0,
"max_fg": 1300.0,
"samples": 318
},
"Spirit": {
"median_fg": 25.0,
"avg_fg": 78.3,
"min_fg": 0.0,
"max_fg": 1300.0,
"samples": 237
},
"Wind": {
"median_fg": 30.0,
"avg_fg": 175.0,
"min_fg": 0.0,
"max_fg": 1200.0,
"samples": 60
},
"Arach": {
"median_fg": 40.0,
"avg_fg": 184.2,
"min_fg": 5.0,
"max_fg": 3500.0,
"samples": 210
},
"Sorc Torch": {
"median_fg": 50.0,
"avg_fg": 166.2,
"min_fg": 10.0,
"max_fg": 2500.0,
"samples": 64
},
"Griffon": {
"median_fg": 60.0,
"avg_fg": 254.7,
"min_fg": 5.0,
"max_fg": 3500.0,
"samples": 111
},
"Guardian's Light": {
"median_fg": 50.0,
"avg_fg": 342.6,
"min_fg": 0.0,
"max_fg": 3500.0,
"samples": 236
},
"Skin": {
"median_fg": 30.0,
"avg_fg": 206.8,
"min_fg": 0.0,
"max_fg": 3000.0,
"samples": 242
},
"Viper Gorge": {
"median_fg": 30.0,
"avg_fg": 75.9,
"min_fg": 0.0,
"max_fg": 1200.0,
"samples": 231
},
"Lo Rune": {
"median_fg": 30.0,
"avg_fg": 113.6,
"min_fg": 5.0,
"max_fg": 1500.0,
"samples": 105
},
"Enigma": {
"median_fg": 20.0,
"avg_fg": 182.4,
"min_fg": 0.0,
"max_fg": 1500.0,
"samples": 49
},
"Bone": {
"median_fg": 40.0,
"avg_fg": 244.3,
"min_fg": 0.0,
"max_fg": 3500.0,
"samples": 264
},
"Um Rune": {
"median_fg": 45.0,
"avg_fg": 187.3,
"min_fg": 0.0,
"max_fg": 3500.0,
"samples": 191
},
"Lem Rune": {
"median_fg": 20.0,
"avg_fg": 48.7,
"min_fg": 0.0,
"max_fg": 750.0,
"samples": 126
},
"Crescent": {
"median_fg": 10.0,
"avg_fg": 63.0,
"min_fg": 5.0,
"max_fg": 600.0,
"samples": 70
},
"Ist Rune": {
"median_fg": 45.0,
"avg_fg": 74.9,
"min_fg": 3.0,
"max_fg": 750.0,
"samples": 173
},
"Highlord": {
"median_fg": 25.0,
"avg_fg": 113.0,
"min_fg": 5.0,
"max_fg": 1500.0,
"samples": 70
},
"BK Ring": {
"median_fg": 20.0,
"avg_fg": 70.0,
"min_fg": 0.0,
"max_fg": 1200.0,
"samples": 161
},
"Mal Rune": {
"median_fg": 30.0,
"avg_fg": 67.0,
"min_fg": 3.0,
"max_fg": 750.0,
"samples": 166
},
"Insight": {
"median_fg": 40.0,
"avg_fg": 189.9,
"min_fg": 5.0,
"max_fg": 3500.0,
"samples": 155
},
"Pul Rune": {
"median_fg": 30.0,
"avg_fg": 69.6,
"min_fg": 5.0,
"max_fg": 750.0,
"samples": 113
},
"CTA": {
"median_fg": 50.0,
"avg_fg": 251.5,
"min_fg": 5.0,
"max_fg": 3500.0,
"samples": 171
},
"Blessed": {
"median_fg": 30.0,
"avg_fg": 180.3,
"min_fg": 7.0,
"max_fg": 1200.0,
"samples": 36
},
"Ohm Rune": {
"median_fg": 25.0,
"avg_fg": 39.8,
"min_fg": 3.0,
"max_fg": 400.0,
"samples": 119
},
"Java SK": {
"median_fg": 20.0,
"avg_fg": 47.0,
"min_fg": 1.0,
"max_fg": 450.0,
"samples": 58
},
"Vampire Gaze": {
"median_fg": 10.0,
"avg_fg": 110.6,
"min_fg": 0.0,
"max_fg": 1500.0,
"samples": 80
},
"Giant": {
"median_fg": 20.0,
"avg_fg": 96.1,
"min_fg": 0.0,
"max_fg": 600.0,
"samples": 70
},
"Ber Rune": {
"median_fg": 60.0,
"avg_fg": 253.1,
"min_fg": 5.0,
"max_fg": 1200.0,
"samples": 58
},
"Key": {
"median_fg": 20.0,
"avg_fg": 83.1,
"min_fg": 1.0,
"max_fg": 1200.0,
"samples": 114
},
"Shako": {
"median_fg": 30.0,
"avg_fg": 113.6,
"min_fg": 0.0,
"max_fg": 1500.0,
"samples": 208
},
"Pcomb SK": {
"median_fg": 160.0,
"avg_fg": 685.0,
"min_fg": 5.0,
"max_fg": 3000.0,
"samples": 48
},
"Sacred": {
"median_fg": 35.0,
"avg_fg": 166.1,
"min_fg": 0.0,
"max_fg": 3500.0,
"samples": 244
},
"Light Jewel": {
"median_fg": 80.0,
"avg_fg": 234.0,
"min_fg": 20.0,
"max_fg": 1500.0,
"samples": 39
},
"Fist": {
"median_fg": 20.0,
"avg_fg": 51.7,
"min_fg": 15.0,
"max_fg": 120.0,
"samples": 9
},
"Vex Rune": {
"median_fg": 30.0,
"avg_fg": 97.2,
"min_fg": 5.0,
"max_fg": 1500.0,
"samples": 132
},
"Unid Anni": {
"median_fg": 35.0,
"avg_fg": 45.0,
"min_fg": 3.0,
"max_fg": 150.0,
"samples": 96
},
"Light SK": {
"median_fg": 40.0,
"avg_fg": 383.2,
"min_fg": 5.0,
"max_fg": 3500.0,
"samples": 41
},
"Mara": {
"median_fg": 50.0,
"avg_fg": 146.1,
"min_fg": 5.0,
"max_fg": 1500.0,
"samples": 132
},
"Cold SK": {
"median_fg": 50.0,
"avg_fg": 162.9,
"min_fg": 7.0,
"max_fg": 1200.0,
"samples": 45
},
"Black": {
"median_fg": 30.0,
"avg_fg": 106.7,
"min_fg": 0.0,
"max_fg": 2400.0,
"samples": 199
},
"Jah Rune": {
"median_fg": 60.0,
"avg_fg": 267.5,
"min_fg": 0.0,
"max_fg": 1200.0,
"samples": 87
},
"Troll": {
"median_fg": 30.0,
"avg_fg": 32.1,
"min_fg": 1.0,
"max_fg": 75.0,
"samples": 66
},
"Ward": {
"median_fg": 10.0,
"avg_fg": 21.8,
"min_fg": 1.0,
"max_fg": 50.0,
"samples": 45
},
"White": {
"median_fg": 20.0,
"avg_fg": 37.6,
"min_fg": 3.0,
"max_fg": 300.0,
"samples": 126
},
"Stealth RW": {
"median_fg": 32.5,
"avg_fg": 31.7,
"min_fg": 5.0,
"max_fg": 50.0,
"samples": 24
},
"Doom": {
"median_fg": 50.0,
"avg_fg": 285.3,
"min_fg": 45.0,
"max_fg": 1500.0,
"samples": 30
},
"Gul Rune": {
"median_fg": 30.0,
"avg_fg": 50.4,
"min_fg": 0.0,
"max_fg": 280.0,
"samples": 136
},
"Death's Fathom": {
"median_fg": 110.0,
"avg_fg": 96.7,
"min_fg": 60.0,
"max_fg": 120.0,
"samples": 12
},
"Unid Torch": {
"median_fg": 60.0,
"avg_fg": 49.5,
"min_fg": 5.0,
"max_fg": 120.0,
"samples": 39
},
"Sanctuary": {
"median_fg": 85.0,
"avg_fg": 82.5,
"min_fg": 5.0,
"max_fg": 170.0,
"samples": 24
},
"Grief": {
"median_fg": 50.0,
"avg_fg": 170.2,
"min_fg": 5.0,
"max_fg": 1500.0,
"samples": 56
},
"Night": {
"median_fg": 250.0,
"avg_fg": 209.0,
"min_fg": 20.0,
"max_fg": 700.0,
"samples": 15
},
"5/5 Facet": {
"median_fg": 50.0,
"avg_fg": 313.3,
"min_fg": 10.0,
"max_fg": 1300.0,
"samples": 15
},
"Treachery": {
"median_fg": 20.0,
"avg_fg": 157.0,
"min_fg": 5.0,
"max_fg": 1300.0,
"samples": 30
},
"Pala Torch": {
"median_fg": 50.0,
"avg_fg": 215.6,
"min_fg": 20.0,
"max_fg": 1500.0,
"samples": 43
},
"Bulwark": {
"median_fg": 50.0,
"avg_fg": 440.0,
"min_fg": 50.0,
"max_fg": 1300.0,
"samples": 10
},
"Fortitude": {
"median_fg": 70.0,
"avg_fg": 915.0,
"min_fg": 20.0,
"max_fg": 3500.0,
"samples": 16
},
"Cham Rune": {
"median_fg": 20.0,
"avg_fg": 22.5,
"min_fg": 5.0,
"max_fg": 60.0,
"samples": 53
},
"War Traveler": {
"median_fg": 20.0,
"avg_fg": 33.0,
"min_fg": 5.0,
"max_fg": 80.0,
"samples": 33
},
"Oculus": {
"median_fg": 20.0,
"avg_fg": 29.3,
"min_fg": 5.0,
"max_fg": 90.0,
"samples": 63
},
"Moser": {
"median_fg": 12.5,
"avg_fg": 12.5,
"min_fg": 5.0,
"max_fg": 20.0,
"samples": 16
},
"Hellfire": {
"median_fg": 60.0,
"avg_fg": 53.3,
"min_fg": 20.0,
"max_fg": 80.0,
"samples": 12
},
"Conviction": {
"median_fg": 20.0,
"avg_fg": 13.7,
"min_fg": 1.0,
"max_fg": 20.0,
"samples": 6
},
"Terror Key": {
"median_fg": 5.0,
"avg_fg": 6.7,
"min_fg": 5.0,
"max_fg": 10.0,
"samples": 12
},
"Hate Key": {
"median_fg": 5.0,
"avg_fg": 6.7,
"min_fg": 5.0,
"max_fg": 10.0,
"samples": 12
},
"Dual Leech Ring": {
"median_fg": 50.0,
"avg_fg": 198.8,
"min_fg": 20.0,
"max_fg": 1500.0,
"samples": 16
},
"Cyclone": {
"median_fg": 300.0,
"avg_fg": 306.7,
"min_fg": 20.0,
"max_fg": 600.0,
"samples": 12
},
"Fury": {
"median_fg": 30.0,
"avg_fg": 76.9,
"min_fg": 5.0,
"max_fg": 250.0,
"samples": 32
}
},
"day_37": {},
"day_38": {},
"day_39": {},
"day_40": {},

1202
config/price_dashboard.html Normal file
View File

File diff suppressed because it is too large Load Diff

154
docs/melee_hunt_test_log.md Normal file
View File

@@ -0,0 +1,154 @@
# Paladalla Melee Hunt Test Log
Date: 2026-08-23
Workspace: `C:\Users\alex\my-botty`
Character/profile: `paladalla`
Script: `tools/melee_hunt.py`
## Current State
- Testing was stopped at the user's request.
- No `melee_hunt` Python process was running when checked.
- Paladalla was alive in the Rogue Encampment at the end of the session.
- The four visible belt potions were manually consumed. The last screenshot shows
partial life and an empty belt, so refill and fully heal before the next run.
- The game must not be controlled automatically until the user asks to resume.
Final-state screenshot: [healed_from_belt.png](../log/screenshots/manual/healed_from_belt.png)
## Commands Used
The successful short run used:
```powershell
C:\Users\alex\miniforge3\envs\botty\python.exe tools\melee_hunt.py --debug --button left
```
The later long tests used:
```powershell
C:\Users\alex\miniforge3\envs\botty\python.exe tools\melee_hunt.py --debug --button left --minutes 1440
```
`F12` is intended to stop the script. It worked during one run but was not
reliable while an in-game overlay was open. `Ctrl+C` in the script terminal was
the reliable emergency stop.
## Verified Results
### Initial ten-minute run
- Started outside town and completed its configured timer normally.
- Result: `engaged 107, roamed 15`.
- XP changed from `261/500` to `357/500`.
- Final reported life was `69%`.
- The process then exited because the default duration is ten minutes. Paladalla
later died because nothing was controlling or protecting the character.
### Corpse recovery and level gain
- The death prompt was cleared and the game was cycled through Save and Exit.
- The corpse appeared beside Paladalla in town and was recovered successfully.
- Weapon and attack slots were restored.
- Paladalla was manually routed across the bridge into the Blood Moor.
- Manual movement/attack clicks later produced a level-up. The next script start
read XP as `564/1500`, confirming level 2 and working melee damage.
Evidence:
- [corpse_recovered.png](../log/screenshots/manual/corpse_recovered.png)
- [blood_moor_ready2.png](../log/screenshots/manual/blood_moor_ready2.png)
- [deep_moor_restart.png](../log/screenshots/manual/deep_moor_restart.png)
## Failures Found
### 1. Ten-minute default looked like a crash
The first hunt was not dead or hung. It reached its default ten-minute deadline
and exited cleanly. Long unattended tests must pass an explicit `--minutes`
value or implement a true continuous mode.
### 2. Friendly bridge NPC was treated as motion target
The first 24-hour attempt remained near the town bridge. At the two-minute
checkpoint it showed repeated attacks, full life, and unchanged XP
(`357/500`). The screenshot showed it attacking the friendly rogue near the
bridge. Motion alone cannot distinguish that NPC from a monster.
Evidence: [hunt_2min_checkpoint.png](../log/screenshots/manual/hunt_2min_checkpoint.png)
Workaround used: stop the script and move several screens deeper into the Blood
Moor before restarting. A permanent NPC rejection/stall detector is still
needed.
### 3. Animated HUD controls were treated as targets
After reaching level 2, the pulsing new-stat/new-skill controls appeared above
the original HUD mask. The motion detector could target that region. During the
second long attempt an in-game Loot Filter overlay opened and the script kept
issuing clicks behind it.
Evidence: [restart_live_40s.png](../log/screenshots/manual/restart_live_40s.png)
### 4. No-potion condition was unsafe
Life fell through `18%`, `14%`, `10%`, `8%`, and eventually `0%`. The script
logged `no healing potion in belt!` but continued its loop. Belt detection also
reported no potion even though red potions were visibly present, so belt
detection or its profile assumptions need separate investigation.
### 5. XP OCR became invalid after level-up/UI changes
The log reported XP OCR values such as `'-'` and `''`, then incorrectly printed
`*** LEVEL UP *** now 0/0`. Invalid OCR should not be interpreted as a level-up,
and `xp_last` should retain the last valid sample.
## Code Changes Made
`tools/melee_hunt.py` was changed in two ways:
1. The target scan bottom boundary was raised from `640` to `510`, excluding the
animated level-up/stat controls and the interactive HUD from motion targets.
2. Failure to detect or use a healing potion at the configured health threshold
now raises `SafetyStop`, exits the hunt loop, and prints a safety-stop reason.
The edited script passed:
```powershell
C:\Users\alex\miniforge3\envs\botty\python.exe -m py_compile tools\melee_hunt.py
```
These changes were not runtime-verified after editing because testing was
stopped. The narrower scan may ignore monsters near the bottom of the viewport;
that tradeoff must be checked visually.
## Next Test Checklist
1. Confirm no old hunt process is running.
2. Refill the belt with healing potions and fully heal at Akara.
3. Save and Exit, re-enter, and recover any corpse if one appears.
4. Spend or dismiss the new stat/skill notifications if practical.
5. Walk across the bridge and several screens into the Blood Moor before launch.
6. Start with a short five-minute test, not a 24-hour run:
```powershell
C:\Users\alex\miniforge3\envs\botty\python.exe tools\melee_hunt.py --debug --button left --minutes 5
```
7. Capture and inspect screenshots at launch, two minutes, and five minutes.
8. At each checkpoint verify: outside town, no modal open, attacking a hostile,
XP increasing, life stable, movement occurring, and no repeated target lock.
9. Force a controlled low-health/no-potion condition only after normal combat is
proven, and verify `SAFETY STOP` exits immediately.
10. Use `Ctrl+C` if any modal opens, XP remains unchanged across two checkpoints,
life drops rapidly, or the character approaches the town bridge NPC.
## Follow-up Engineering Work
- Add a stall detector based on unchanged XP plus repeated target coordinates.
- Temporarily blacklist a target position after excessive swings without XP.
- Reject invalid XP samples instead of converting them into `0/0` level-ups.
- Investigate why `belt.drink_potion("health", ...)` missed visible red potions.
- Detect death and modal overlays explicitly and stop before sending more input.
- Consider an explicit `--continuous` option instead of using a large minute
count.

5
src/input_layer/win_input.py
View File

@@ -300,6 +300,11 @@ def mouse_move(x: int, y: int):
Logger.warning(f"SetCursorPos failed for ({target_x}, {target_y}); continuing with current cursor position")
except Exception:
pass
# Ensure the cursor is exactly on target before any click (SetCursorPos can
# land 1-2px off on some DPI/virtual-desktop setups; D2R UI hitboxes are tight).
cur_x, cur_y = get_cursor_pos()
if abs(cur_x - target_x) > 0 or abs(cur_y - target_y) > 0:
user32.SetCursorPos(target_x, target_y)
def mouse_down(button: str = "left"):
"""Press mouse button down."""

BIN
tools/assets/templates/a5_wp.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 24 KiB

313
tools/build_price_dashboard.py Normal file
View File

@@ -0,0 +1,313 @@
#!/usr/bin/env python
"""Generate D2R Price Dashboard - FG (d2jsp) + Rune Trade Prices (Traderie).
Reads:
config/daily_prices.json - merged daily prices (traderie + fg)
config/daily_prices_history.json - history of daily prices
config/fg_daily_estimates.json - FG estimates from d2jsp
config/traderie_prices.json - latest traderie scrape
Writes:
config/price_dashboard.html - self-contained HTML dashboard
"""
import json
import html
from pathlib import Path
from datetime import datetime
REPO = Path(r"C:\Users\alex\my-botty")
CONFIG = REPO / "config"
OUT = CONFIG / "price_dashboard.html"
# Rune value ladder (relative to El=1) for normalizing trade prices
RUNE_VALUES = {
"el rune": 1, "eld rune": 1, "tir rune": 1, "nef rune": 1,
"eth rune": 1, "ith rune": 1,
"tal rune": 2, "ral rune": 3, "ort rune": 4, "amn rune": 5,
"sol rune": 6, "shael rune": 3, "dol rune": 3, "hel rune": 3,
"io rune": 4, "ko rune": 4, "lum rune": 4,
"fal rune": 7, "lem rune": 8, "pul rune": 9, "um rune": 10,
"mal rune": 12, "ist rune": 15, "gul rune": 18, "vex rune": 22,
"ohm rune": 25, "lo rune": 30, "cham rune": 35,
"jah rune": 50, "ber rune": 60, "koh rune": 70,
"zod rune": 80,
}
CHARM_VALUES = {"small charm": 0.5, "medium charm": 1, "large charm": 2, "grand charm": 5}
CURRENCY_VALUES = {**RUNE_VALUES, **CHARM_VALUES}
def load_json(name, default=None):
p = CONFIG / name
if p.exists():
try:
return json.loads(p.read_text(encoding="utf-8"))
except Exception as e:
print(f"WARNING: could not load {name}: {e}")
return default if default is not None else {}
def normalize_trade_price(currencies: dict) -> float:
"""Convert a currency dict {name: {count, median...}} to a normalized value (El=1)."""
total = 0.0
for cur, stats in currencies.items():
val = CURRENCY_VALUES.get(cur.lower(), 0)
count = stats.get("count", 1) if isinstance(stats, dict) else 1
total += val * count
return total
def build_dashboard():
daily = load_json("daily_prices.json", {})
history = load_json("daily_prices_history.json", [])
fg_est = load_json("fg_daily_estimates.json", {})
traderie = load_json("traderie_prices.json", {})
prices = daily.get("prices", {})
date = daily.get("date", "unknown")
generated = daily.get("generated_at", "")
n_listings = daily.get("traderie_listings_scanned", 0)
# Build item rows: name, fg_price, trade_price (normalized), trade_detail, listings, fg_name
rows = []
for name, data in prices.items():
tr = data.get("traderie")
fg = data.get("fg")
fg_name = data.get("fg_name")
trade_norm = None
trade_detail = ""
listings = 0
if tr:
listings = tr.get("total_listings", 0)
curs = tr.get("currencies", {})
overall = tr.get("overall", {})
trade_norm = overall.get("median")
# Build human-readable detail
parts = []
for cur, stats in curs.items():
cnt = stats.get("count", 1) if isinstance(stats, dict) else 1
med = stats.get("median", "?") if isinstance(stats, dict) else "?"
parts.append(f"{cnt}x {cur} ({med})" if cnt > 1 else f"{cur} ({med})")
trade_detail = " + ".join(parts)
rows.append({
"name": name,
"fg": fg,
"fg_name": fg_name,
"trade_norm": trade_norm,
"trade_detail": trade_detail,
"listings": listings,
"has_fg": fg is not None,
"has_trade": tr is not None,
})
# Sort: items with both first, then by max price
def sort_key(r):
fg_v = r["fg"] if isinstance(r["fg"], (int, float)) else 0
tr_v = r["trade_norm"] if isinstance(r["trade_norm"], (int, float)) else 0
return (-(1 if r["has_fg"] and r["has_trade"] else 0), -(max(fg_v, tr_v)))
rows.sort(key=sort_key)
# History for sparklines: date -> {item: value}
hist_by_date = {}
for entry in history:
d = entry.get("date")
if d:
hist_by_date[d] = entry.get("prices", {})
hist_dates = sorted(hist_by_date.keys())
# FG estimates (d2jsp forum)
fg_estimates = fg_est.get("estimates", {})
topics_scanned = fg_est.get("topics_scanned", 0)
# Traderie rune price summary (what runes are worth in listings)
rune_summary = {}
for name, data in prices.items():
tr = data.get("traderie")
if not tr:
continue
for cur, stats in tr.get("currencies", {}).items():
if cur.lower() in RUNE_VALUES or cur.lower() in CHARM_VALUES:
cnt = stats.get("count", 0) if isinstance(stats, dict) else 0
rune_summary[cur] = rune_summary.get(cur, 0) + cnt
# Build HTML
now = datetime.now().strftime("%Y-%m-%d %H:%M")
html_parts = []
html_parts.append(f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>D2R Price Dashboard</title>
<style>
:root {{
--bg: #0d1117; --panel: #161b22; --border: #30363d;
--text: #e6edf3; --muted: #8b949e; --accent: #58a6ff;
--green: #3fb950; --red: #f85149; --yellow: #d29922; --purple: #bc8cff;
}}
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
body {{ background: var(--bg); color: var(--text); font-family: 'Segoe UI', system-ui, sans-serif; padding: 20px; }}
h1 {{ font-size: 1.6em; margin-bottom: 4px; }}
.subtitle {{ color: var(--muted); font-size: 0.9em; margin-bottom: 20px; }}
.stats {{ display: flex; gap: 12px; flex-wrap: wrap; margin-bottom: 20px; }}
.stat {{ background: var(--panel); border: 1px solid var(--border); border-radius: 8px; padding: 12px 18px; min-width: 140px; }}
.stat .label {{ color: var(--muted); font-size: 0.75em; text-transform: uppercase; letter-spacing: 0.5px; }}
.stat .value {{ font-size: 1.5em; font-weight: 700; margin-top: 2px; }}
.stat .value.green {{ color: var(--green); }}
.stat .value.blue {{ color: var(--accent); }}
.stat .value.purple {{ color: var(--purple); }}
.stat .value.yellow {{ color: var(--yellow); }}
.section {{ margin-bottom: 24px; }}
.section h2 {{ font-size: 1.1em; margin-bottom: 10px; color: var(--accent); border-bottom: 1px solid var(--border); padding-bottom: 6px; }}
table {{ width: 100%; border-collapse: collapse; background: var(--panel); border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }}
th {{ background: #1c2128; color: var(--muted); font-size: 0.75em; text-transform: uppercase; letter-spacing: 0.5px; padding: 10px 12px; text-align: left; position: sticky; top: 0; }}
td {{ padding: 8px 12px; border-top: 1px solid var(--border); font-size: 0.9em; }}
tr:hover td {{ background: #1c2128; }}
.price {{ font-weight: 700; font-variant-numeric: tabular-nums; }}
.price.fg {{ color: var(--green); }}
.price.trade {{ color: var(--purple); }}
.badge {{ display: inline-block; padding: 2px 8px; border-radius: 10px; font-size: 0.7em; font-weight: 600; }}
.badge.fg {{ background: rgba(63,185,80,0.15); color: var(--green); }}
.badge.trade {{ background: rgba(188,140,255,0.15); color: var(--purple); }}
.badge.both {{ background: rgba(88,166,255,0.15); color: var(--accent); }}
.detail {{ color: var(--muted); font-size: 0.8em; }}
.num {{ text-align: right; font-variant-numeric: tabular-nums; }}
.search {{ width: 100%; padding: 10px 14px; background: var(--panel); border: 1px solid var(--border); border-radius: 8px; color: var(--text); font-size: 0.95em; margin-bottom: 12px; }}
.search:focus {{ outline: none; border-color: var(--accent); }}
.rune-grid {{ display: grid; grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); gap: 8px; }}
.rune-card {{ background: var(--panel); border: 1px solid var(--border); border-radius: 8px; padding: 10px 14px; }}
.rune-card .name {{ font-weight: 600; font-size: 0.9em; }}
.rune-card .count {{ color: var(--muted); font-size: 0.8em; }}
.rune-card .value {{ color: var(--yellow); font-weight: 700; }}
.table-wrap {{ max-height: 600px; overflow-y: auto; border-radius: 8px; }}
.footer {{ color: var(--muted); font-size: 0.75em; margin-top: 20px; text-align: center; }}
.hidden {{ display: none; }}
</style>
</head>
<body>
<h1>⚔️ D2R Price Dashboard</h1>
<div class="subtitle">FG prices (d2jsp forum) + Rune trade prices (Traderie) &mdash; {html.escape(date)} &mdash; generated {now}</div>
<div class="stats">
<div class="stat"><div class="label">Items Tracked</div><div class="value blue">{len(rows)}</div></div>
<div class="stat"><div class="label">With FG Price</div><div class="value green">{sum(1 for r in rows if r['has_fg'])}</div></div>
<div class="stat"><div class="label">With Trade Price</div><div class="value purple">{sum(1 for r in rows if r['has_trade'])}</div></div>
<div class="stat"><div class="label">Both Sources</div><div class="value">{sum(1 for r in rows if r['has_fg'] and r['has_trade'])}</div></div>
<div class="stat"><div class="label">Traderie Listings</div><div class="value yellow">{n_listings}</div></div>
<div class="stat"><div class="label">d2jsp Topics</div><div class="value">{topics_scanned}</div></div>
<div class="stat"><div class="label">History Days</div><div class="value">{len(hist_dates)}</div></div>
</div>
<div class="section">
<h2>📊 All Items (FG + Trade)</h2>
<input type="text" class="search" id="search" placeholder="Search items... (e.g. Mara, Ist, Torch, Shako)" oninput="filterTable()">
<div class="table-wrap">
<table id="mainTable">
<thead><tr>
<th>Item</th>
<th class="num">FG Price</th>
<th class="num">Trade Value</th>
<th>Trade Detail</th>
<th class="num">Listings</th>
<th>Source</th>
</tr></thead>
<tbody>""")
for r in rows:
fg_cls = "price fg" if r["has_fg"] and isinstance(r["fg"], (int, float)) else ""
fg_txt = f"{r['fg']}" if isinstance(r["fg"], (int, float)) else "—"
tr_cls = "price trade" if r["has_trade"] else ""
tr_txt = f"~{r['trade_norm']:.0f}" if r["trade_norm"] is not None else "—"
if r["has_fg"] and r["has_trade"]:
badge = '<span class="badge both">FG+Trade</span>'
elif r["has_fg"]:
badge = '<span class="badge fg">FG</span>'
elif r["has_trade"]:
badge = '<span class="badge trade">Trade</span>'
else:
badge = ""
name_esc = html.escape(r["name"])
detail_esc = html.escape(r["trade_detail"])
html_parts.append(f""" <tr>
<td><strong>{name_esc}</strong>{f' <span class="detail">({html.escape(r["fg_name"])})</span>' if r['fg_name'] and r['fg_name'] != r['name'] else ''}</td>
<td class="num {fg_cls}">{fg_txt}</td>
<td class="num {tr_cls}">{tr_txt}</td>
<td class="detail">{detail_esc}</td>
<td class="num">{r['listings'] if r['has_trade'] else '—'}</td>
<td>{badge}</td>
</tr>""")
html_parts.append(""" </tbody>
</table>
</div>
</div>""")
# Rune summary section
if rune_summary:
html_parts.append("""
<div class="section">
<h2>🪙 Rune/Charm Demand (Traderie listings as currency)</h2>
<div class="rune-grid">""")
for cur in sorted(rune_summary.keys(), key=lambda c: -rune_summary[c]):
val = CURRENCY_VALUES.get(cur.lower(), 0)
html_parts.append(f""" <div class="rune-card">
<div class="name">{html.escape(cur)}</div>
<div class="count">{rune_summary[cur]} listings</div>
<div class="value">≈{val} El</div>
</div>""")
html_parts.append(""" </div>
</div>""")
# History section
if len(hist_dates) > 1:
html_parts.append(f"""
<div class="section">
<h2>📈 Price History ({len(hist_dates)} days)</h2>
<div class="table-wrap">
<table>
<thead><tr><th>Date</th><th class="num">Items</th><th class="num">FG Items</th><th class="num">Trade Items</th></tr></thead>
<tbody>""")
for d in hist_dates:
p = hist_by_date[d]
n_items = len(p)
n_fg = sum(1 for v in p.values() if v.get("fg") is not None)
n_tr = sum(1 for v in p.values() if v.get("traderie") is not None)
html_parts.append(f""" <tr><td>{html.escape(d)}</td><td class="num">{n_items}</td><td class="num">{n_fg}</td><td class="num">{n_tr}</td></tr>""")
html_parts.append(""" </tbody>
</table>
</div>
</div>""")
html_parts.append(f"""
<div class="footer">
Sources: d2jsp forums (FG prices) + traderie.com (trade prices) &mdash; auto-scraped daily at 06:00<br>
Trade Value = normalized to El Rune (1 El = 1). FG = Forum Gold (d2jsp).<br>
Last scrape: {html.escape(generated)}
</div>
<script>
function filterTable() {{
const q = document.getElementById('search').value.toLowerCase();
const rows = document.querySelectorAll('#mainTable tbody tr');
rows.forEach(row => {{
const text = row.textContent.toLowerCase();
row.classList.toggle('hidden', q && !text.includes(q));
}});
}}
</script>
</body>
</html>""")
OUT.write_text("".join(html_parts), encoding="utf-8")
print(f"Dashboard written to {OUT}")
print(f" Items: {len(rows)}")
print(f" With FG: {sum(1 for r in rows if r['has_fg'])}")
print(f" With Trade: {sum(1 for r in rows if r['has_trade'])}")
print(f" Both: {sum(1 for r in rows if r['has_fg'] and r['has_trade'])}")
print(f" History days: {len(hist_dates)}")
print(f" Rune currencies: {len(rune_summary)}")
if __name__ == "__main__":
build_dashboard()

25
tools/melee_hunt.py
View File

@@ -33,7 +33,9 @@ def main():
ap = argparse.ArgumentParser()
ap.add_argument("--minutes", type=float, default=10.0)
ap.add_argument("--stop-key", default="f12")
ap.add_argument("--swings", type=int, default=5, help="clicks per target before rescanning")
ap.add_argument("--swings", type=int, default=10, help="clicks per target before rescanning")
ap.add_argument("--button", default="right", choices=["left", "right"],
help="mouse button to attack with (default: right)")
ap.add_argument("--min-area", type=int, default=80, help="smallest motion blob to treat as a monster")
ap.add_argument("--max-area", type=int, default=9000, help="largest motion blob to accept")
ap.add_argument("--debug", action="store_true", help="save an annotated scan image each cycle")
@@ -75,7 +77,10 @@ def main():
W, H = 1280, 720
CX, CY = W // 2, 340 # character sits slightly above centre
SELF_R = 85 # ignore her own animation
HUD_TOP, HUD_BOTTOM = 70, 640 # ignore the name banner and the control bar
# The level-up/stat controls pulse above the main control bar and look like
# monsters to a motion detector. Keep the entire interactive HUD out of the
# target image so a click can never open a panel during combat.
HUD_TOP, HUD_BOTTOM = 70, 510
fm = Config().char["force_move"]
stand_still = Config().char["stand_still"]
@@ -111,6 +116,9 @@ def main():
cv2.imwrite("log/screenshots/manual/hunt_debug.png", dbg)
return out
class SafetyStop(RuntimeError):
pass
def check_health():
"""Drink a healing potion if life has dropped to the threshold.
@@ -126,9 +134,14 @@ def main():
print(f" LIFE {hp*100:.0f}% <= {opts.heal_at*100:.0f}% — drinking potion")
try:
if not belt.drink_potion("health", stats=[hp, 1.0]):
print(" no healing potion in belt!")
raise SafetyStop(
f"life is {hp*100:.0f}% and no healing potion was detected")
except SafetyStop:
raise
except Exception as e:
print(f" potion failed: {e}")
raise SafetyStop(
f"life is {hp*100:.0f}% and potion use failed") from e
wait(0.4, 0.6)
return hp
@@ -188,9 +201,9 @@ def main():
if stopping():
break
mouse.move(mx, my, randomize=6, delay_factor=[0.15, 0.25])
mouse.click(button="left")
mouse.click(button=opts.button)
wait(0.22, 0.32)
if i and i % 2 == 0:
if i and i % 4 == 0:
hp = check_health()
finally:
keyboard.send(stand_still, do_press=False)
@@ -240,6 +253,8 @@ def main():
f"life={hp_txt} xp={xp_txt} "
f"{max(0, deadline - time.time()):5.0f}s left ")
print(CR + status, end="", flush=True)
except SafetyStop as e:
print(f"\nSAFETY STOP: {e}")
except KeyboardInterrupt:
print("\ninterrupted")

257
tools/push_prices_to_db.py Normal file
View File

@@ -0,0 +1,257 @@
#!/usr/bin/env python
"""Push D2R price data to MariaDB on .96.
Reads config/daily_prices.json + config/fg_prices.json from the repo,
pushes all prices to the d2r_prices database on 192.168.1.96:3306.
FG prices come from fg_prices.json rules (name → fg value).
Trade prices come from daily_prices.json (traderie data with raw currency detail).
Usage:
python tools/push_prices_to_db.py
"""
import json
import re
import sys
from datetime import datetime
from pathlib import Path
try:
import pymysql
except ImportError:
print("ERROR: pymysql not installed. Run: pip install pymysql")
sys.exit(1)
REPO = Path(__file__).resolve().parent.parent
CONFIG = REPO / "config"
DB_CONFIG = {
"host": "192.168.1.96",
"port": 3306,
"user": "d2r",
"password": "D2rPrices@2026!",
"database": "d2r_prices",
"charset": "utf8mb4",
}
def load_json(name, default=None):
p = CONFIG / name
if p.exists():
try:
return json.loads(p.read_text(encoding="utf-8"))
except Exception as e:
print(f"WARNING: could not load {name}: {e}")
return default if default is not None else {}
def build_fg_lookup(fg_config):
"""Build a lookup from fg_prices.json rules: item_name → fg_price."""
lookup = {}
rules = fg_config.get("rules", [])
for rule in rules:
name = rule.get("name", "")
fg = rule.get("fg")
if name and isinstance(fg, (int, float)):
lookup[name.lower()] = float(fg)
return lookup
def match_fg(item_name, fg_lookup):
"""Try to match an item name to an FG price rule."""
# Exact match first
key = item_name.lower()
if key in fg_lookup:
return fg_lookup[key]
# Try without common suffixes
for suffix in [" rune", " charm", " jewel", " amulet"]:
if key.endswith(suffix):
base = key[:-len(suffix)]
if base in fg_lookup:
return fg_lookup[base]
return None
def format_trade_detail(tr):
"""Format trade detail as '2 Jah Rune + 1 Ber Rune' (not El Rune normalized)."""
if not tr:
return ""
parts = []
for cur, stats in tr.get("currencies", {}).items():
if isinstance(stats, dict):
cnt = stats.get("count", 1)
med = stats.get("median", "?")
if cnt > 1:
parts.append(f"{cnt} {cur} ({med})")
else:
parts.append(f"{cur} ({med})")
else:
parts.append(f"{cur} ({stats})")
return " + ".join(parts)
def extract_currency_prices(prices):
"""Extract per-currency median prices from all trade listings.
Returns: {currency_name: {"median": float, "listings": int}}
"""
currency_data = {}
for name, data in prices.items():
tr = data.get("traderie")
if not tr:
continue
for cur, stats in tr.get("currencies", {}).items():
if not isinstance(stats, dict):
continue
med = stats.get("median")
cnt = stats.get("count", 0)
if med is None or not isinstance(med, (int, float)):
continue
if cur not in currency_data:
currency_data[cur] = {"prices": [], "listings": 0}
currency_data[cur]["prices"].append(float(med))
currency_data[cur]["listings"] += cnt
# Compute median for each currency
result = {}
for cur, d in currency_data.items():
if d["prices"]:
sorted_p = sorted(d["prices"])
n = len(sorted_p)
median = sorted_p[n // 2] if n % 2 == 1 else (sorted_p[n // 2 - 1] + sorted_p[n // 2]) / 2
result[cur] = {"median": median, "listings": d["listings"]}
return result
def push():
daily = load_json("daily_prices.json", {})
fg_config = load_json("fg_prices.json", {})
if not daily:
print("ERROR: daily_prices.json not found or empty")
sys.exit(1)
prices = daily.get("prices", {})
date = daily.get("date", datetime.now().strftime("%Y-%m-%d"))
generated = daily.get("generated_at", "")
if generated and "T" in generated:
generated = generated.replace("T", " ").split("+")[0].split(".")[0]
n_listings = daily.get("traderie_listings_scanned", 0)
# Build FG lookup from rules
fg_lookup = build_fg_lookup(fg_config)
print(f"FG rules loaded: {len(fg_lookup)}")
# Build rows
rows = []
n_fg = n_trade = n_both = 0
for name, data in prices.items():
tr = data.get("traderie")
fg_data = data.get("fg")
# FG price: try merged data first, then rule match
fg_price = None
if isinstance(fg_data, (int, float)):
fg_price = float(fg_data)
else:
fg_price = match_fg(name, fg_lookup)
# Trade: keep raw currency detail, NOT El Rune normalized
trade_value = None
trade_detail = ""
listings = 0
if tr:
listings = tr.get("total_listings", 0)
trade_detail = format_trade_detail(tr)
# Use the primary currency's median as trade_value (not El Rune)
currencies = tr.get("currencies", {})
if currencies:
# Pick the most common currency
primary = max(currencies.items(), key=lambda x: x[1].get("count", 0) if isinstance(x[1], dict) else 0)
cur_name, cur_stats = primary
if isinstance(cur_stats, dict) and isinstance(cur_stats.get("median"), (int, float)):
trade_value = float(cur_stats["median"])
has_fg = fg_price is not None
has_trade = bool(trade_detail)
if has_fg:
n_fg += 1
if has_trade:
n_trade += 1
if has_fg and has_trade:
n_both += 1
source = "both" if (has_fg and has_trade) else ("fg" if has_fg else "trade")
rows.append((
date, name, fg_price, None, trade_value,
trade_detail, listings, int(has_fg), int(has_trade), source,
))
# Add FG-only items that aren't in the Traderie data
# fg_lookup keys are lowercase, so we need the original names
for rule in fg_config.get("rules", []):
rule_name = rule.get("name", "")
rule_fg = rule.get("fg")
if not rule_name or not isinstance(rule_fg, (int, float)):
continue
# Check if already added (case-insensitive)
if any(r[1].lower() == rule_name.lower() for r in rows):
continue
rows.append((
date, rule_name, float(rule_fg), None, None,
"", 0, 1, 0, "fg",
))
n_fg += 1
# Connect and push
conn = pymysql.connect(**DB_CONFIG)
try:
with conn.cursor() as cur:
cur.executemany("""
INSERT INTO prices (scrape_date, item_name, fg_price, fg_name,
trade_value, trade_detail, listings, has_fg, has_trade, source)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE
fg_price=VALUES(fg_price), fg_name=VALUES(fg_name),
trade_value=VALUES(trade_value), trade_detail=VALUES(trade_detail),
listings=VALUES(listings), has_fg=VALUES(has_fg),
has_trade=VALUES(has_trade), source=VALUES(source)
""", rows)
cur.execute("""
INSERT INTO scrape_meta (scrape_date, generated_at, traderie_listings,
d2jsp_topics, items_total, items_fg, items_trade, items_both)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE
generated_at=VALUES(generated_at), traderie_listings=VALUES(traderie_listings),
d2jsp_topics=VALUES(d2jsp_topics), items_total=VALUES(items_total),
items_fg=VALUES(items_fg), items_trade=VALUES(items_trade),
items_both=VALUES(items_both)
""", (date, generated, n_listings, 0, len(rows), n_fg, n_trade, n_both))
conn.commit()
# Push currency prices
currency_prices = extract_currency_prices(prices)
if currency_prices:
cur_rows = [(date, cur, d["median"], d["listings"]) for cur, d in currency_prices.items()]
cur2 = conn.cursor()
cur2.executemany("""
INSERT INTO currency_prices (scrape_date, currency_name, median_price, listings)
VALUES (%s, %s, %s, %s)
ON DUPLICATE KEY UPDATE
median_price=VALUES(median_price), listings=VALUES(listings)
""", cur_rows)
conn.commit()
print(f" Currency prices: {len(cur_rows)} currencies")
print(f"Pushed {len(rows)} prices for {date} to 192.168.1.96")
print(f" FG: {n_fg}, Trade: {n_trade}, Both: {n_both}")
print(f" Traderie listings: {n_listings}")
finally:
conn.close()
if __name__ == "__main__":
push()

40
tools/run_continuous_scrape.bat Normal file
View File

@@ -0,0 +1,40 @@
@echo off
REM Continuous D2R Price Scrape - runs all night
REM Scrape every 30 minutes, push to .96
setlocal
set PYTHON=C:\Python313\python.exe
set REPO=C:\Users\alex\my-botty
set LOG=%REPO%\log\fg_scrape_continuous.log
set INTERVAL=1800 REM 30 minutes in seconds
echo ==========================================>> "%LOG%"
echo [%date% %time%] Starting continuous price scrape>> "%LOG%"
echo ==========================================>> "%LOG%"
:loop
cd /d "%REPO%"
echo [%date% %time%] Running scrape cycle...>> "%LOG%"
REM Scrape d2jsp
"%PYTHON%" tools\playwright_fg_scraper.py --max-forum-pages 40 --max-topics 800 --delay-s 0.5 >> "%LOG%" 2>&1
REM Parse d2jsp
"%PYTHON%" tools\fg_market_scraper.py --offline-dir data/d2jsp_pages --ladder-start-date 2026-05-12 --days 60 >> "%LOG%" 2>&1
REM Scrape Traderie
"%PYTHON%" tools\traderie_price_scraper.py >> "%LOG%" 2>&1
REM Push to .96
scp -o StrictHostKeyChecking=no -r data/d2jsp_pages/* alex@192.168.1.96:/tmp/d2jsp_pages/ >> "%LOG%" 2>&1
scp -o StrictHostKeyChecking=no config/daily_prices.json alex@192.168.1.96:/tmp/ >> "%LOG%" 2>&1
scp -o StrictHostKeyChecking=no config/fg_daily_estimates.json alex@192.168.1.96:/tmp/ >> "%LOG%" 2>&1
ssh -o StrictHostKeyChecking=no alex@192.168.1.96 "sudo mkdir -p /opt/d2r-prices/data/d2jsp_pages /opt/d2r-prices/config && sudo cp /tmp/d2jsp_pages/* /opt/d2r-prices/data/d2jsp_pages/ 2>/dev/null; sudo cp /tmp/daily_prices.json /opt/d2r-prices/config/ 2>/dev/null; sudo cp /tmp/fg_daily_estimates.json /opt/d2r-prices/config/ 2>/dev/null; sudo rm -rf /tmp/d2jsp_pages /tmp/daily_prices.json /tmp/fg_daily_estimates.json; sudo /opt/d2r-prices/run_scrape.sh" >> "%LOG%" 2>&1
echo [%date% %time%] Cycle complete. Waiting %INTERVAL% seconds...>> "%LOG%"
REM Wait 30 minutes
timeout /t %INTERVAL% /nobreak >nul
goto loop

37
tools/run_fg_scrape.bat Normal file
View File

@@ -0,0 +1,37 @@
@echo off
REM Daily D2R Price Pipeline - .211
REM Scrape d2jsp + Traderie -> push data to .96 -> .96 runs DB + dashboard
setlocal
set PYTHON=C:\Python313\python.exe
set REPO=C:\Users\alex\my-botty
set LOG=%REPO%\log\fg_scrape.log
echo ==========================================>> "%LOG%"
echo [%date% %time%] Starting daily price scrape>> "%LOG%"
echo ==========================================>> "%LOG%"
cd /d "%REPO%"
echo [1/3] Scraping d2jsp (40 pages, 800 topics)... >> "%LOG%"
"%PYTHON%" tools\playwright_fg_scraper.py --max-forum-pages 40 --max-topics 800 --delay-s 0.5 >> "%LOG%" 2>&1
if errorlevel 1 (
echo [ERROR] d2jsp scrape failed >> "%LOG%"
goto :end
)
echo [2/3] Scraping Traderie... >> "%LOG%"
"%PYTHON%" tools\traderie_price_scraper.py >> "%LOG%" 2>&1
echo [3/3] Pushing data to .96 and running pipeline... >> "%LOG%"
REM Push d2jsp HTML files
scp -o StrictHostKeyChecking=no -r data/d2jsp_pages/* alex@192.168.1.96:/tmp/d2jsp_pages/ >> "%LOG%" 2>&1
REM Push Traderie + FG data
scp -o StrictHostKeyChecking=no config/daily_prices.json alex@192.168.1.96:/tmp/ >> "%LOG%" 2>&1
scp -o StrictHostKeyChecking=no config/fg_daily_estimates.json alex@192.168.1.96:/tmp/ >> "%LOG%" 2>&1
REM Move to correct location and run .96 pipeline
ssh -o StrictHostKeyChecking=no alex@192.168.1.96 "sudo mkdir -p /opt/d2r-prices/data/d2jsp_pages /opt/d2r-prices/config && sudo cp /tmp/d2jsp_pages/* /opt/d2r-prices/data/d2jsp_pages/ 2>/dev/null; sudo cp /tmp/daily_prices.json /opt/d2r-prices/config/ 2>/dev/null; sudo cp /tmp/fg_daily_estimates.json /opt/d2r-prices/config/ 2>/dev/null; sudo rm -rf /tmp/d2jsp_pages /tmp/daily_prices.json /tmp/fg_daily_estimates.json; sudo /opt/d2r-prices/run_scrape_96.sh" >> "%LOG%" 2>&1
:end
echo [%date% %time%] Pipeline complete >> "%LOG%"
echo Done.