alex/newolddkprice
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add --from flag: travel distance from your location
Some checks are pending
Python application / build (push) Waiting to run
Details

Browse Source 
- Geocodes listing locations via Photon (Komoot) API - no rate limits
- Shows distance in km next to location in all output formats
- Uses geodesic distance (geopy) for accurate km calculation
- Caches geocoded locations to avoid repeated API calls
- Works with --top, --compare, --verdict, and default output
- Example: --from "Farum" shows 'København V (20 km)', 'Korsør (94 km)'
- Added geopy to requirements.txt
This commit is contained in:
Alex
2026-08-28 16:04:11 +02:00
parent 33fdd4dff8
commit 243bc9b6f5
3 changed files with 83 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -47,6 +47,7 @@ Output:
| `--format` | `text` (default), `markdown`, or `grid` |
| `--top N` | Show top N results |
| `--min-price` / `--max-price` | Filter by price (e.g. `--min-price 2000`) |
| `--from` | Your location for distance calculation (e.g. `--from Farum`) |
| `--json` | Output raw JSON |
| `--engine selenium` | Use Selenium instead of requests (for JS-heavy pages) |
@@ -62,6 +63,9 @@ python3 dba_pricerunner_scraper.py "samsung galaxy s24" --pricerunner --compare
# Filter out accessories: only look at items 2000–5000 kr
python3 dba_pricerunner_scraper.py "playstation 5 konsol" --pricerunner --compare --verdict --min-price 2000 --max-price 5000
# Show distances from your location (e.g. Farum)
python3 dba_pricerunner_scraper.py "iphone 15" --from "Farum" --pricerunner --compare --verdict
# ASCII grid for terminal
python3 dba_pricerunner_scraper.py "xbox series x konsol" --pricerunner --compare --verdict --format grid

105
dba_pricerunner_scraper.py
View File

@@ -20,6 +20,40 @@ import requests
from bs4 import BeautifulSoup
from urllib.parse import quote_plus, urljoin
try:
from geopy.distance import geodesic
import requests as _requests
_GEO_CACHE = {}
def _geocode(name):
if not name:
return None
if name in _GEO_CACHE:
return _GEO_CACHE[name]
try:
r = _requests.get(
'https://photon.komoot.io/api/',
params={'q': f'{name}, Danmark'},
headers={'User-Agent': 'dba_pricerunner/1.0'},
timeout=10
)
if r.status_code == 200:
data = r.json()
features = data.get('features', [])
if features:
coords = features[0]['geometry']['coordinates']
result = (coords[1], coords[0]) # [lon, lat] -> (lat, lon)
else:
result = None
else:
result = None
except Exception:
result = None
_GEO_CACHE[name] = result
return result
except ImportError:
def _geocode(name):
return None
try:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service as ChromeService
@@ -118,26 +152,10 @@ def extract_price_string(text):
# DBA scraping
# ──────────────────────────────────────────────────────────────
def _parse_dba_article(el):
def _parse_dba_article(el, origin_coords=None):
"""Extract title, price, url, location from a DBA <article> element.
DBA's current markup:
<article>
<a class="sf-search-ad-link" href="/recommerce/forsale/item/123"></a> ← overlay link
<div class="m-8 ...">
<div>
<div class="flex justify-between ..."><span>3.500 kr.</span></div> ← price
<h2 class="...">Title text</h2> ← title (no <a> inside!)
<div class="flex flex-wrap mt-4 text-xs"><span>Category</span></div>
</div>
<div class="flex flex-col">
<div class="text-xs s-text-subtle ...">
<span class="whitespace-nowrap truncate mr-8">Location</span> ← location
<span class="whitespace-nowrap">5 dage</span>
</div>
</div>
</div>
</article>
If origin_coords is provided, also computes travel distance from origin.
"""
# Title: h2 text directly (the h2 has NO child <a> in current markup)
h2 = el.select_one('h2')
@@ -175,6 +193,13 @@ def _parse_dba_article(el):
if sp and sp.get_text(strip=True):
location = sp.get_text(strip=True)
# Distance from origin (if provided)
distance_km = None
if origin_coords and location:
coords = _geocode(location)
if coords:
distance_km = geodesic(origin_coords, coords).km
price_num = normalize_price(price)
return {
"site": "dba",
@@ -183,11 +208,15 @@ def _parse_dba_article(el):
"price_num": price_num,
"url": link,
"location": location,
"distance_km": distance_km,
}
def search_dba_requests(query, max_results=10):
def search_dba_requests(query, max_results=10, origin=None):
"""Search dba.dk (requests) and return list of dicts with title, price, url, location."""
origin_coords = None
if origin:
origin_coords = _geocode(origin)
q = quote_plus(query)
url = f"https://www.dba.dk/recommerce/forsale/search?q={q}"
r = requests.get(url, headers=HEADERS, timeout=15)
@@ -200,17 +229,20 @@ def search_dba_requests(query, max_results=10):
# skip skeleton/placeholder articles (no text content)
if not el.get_text(strip=True):
continue
item = _parse_dba_article(el)
item = _parse_dba_article(el, origin_coords)
if item['title']:
items.append(item)
time.sleep(1)
return items
def search_dba_selenium(query, max_results=10, headless=True):
def search_dba_selenium(query, max_results=10, headless=True, origin=None):
"""Render DBA with Selenium and extract the same fields as requests path."""
if webdriver is None:
raise ScraperError('Selenium or webdriver-manager not installed')
origin_coords = None
if origin:
origin_coords = _geocode(origin)
opts = ChromeOptions()
if headless:
opts.add_argument('--headless=new')
@@ -228,7 +260,7 @@ def search_dba_selenium(query, max_results=10, headless=True):
for el in results[:max_results]:
if not el.get_text(strip=True):
continue
item = _parse_dba_article(el)
item = _parse_dba_article(el, origin_coords)
if item['title']:
items.append(item)
time.sleep(1)
@@ -441,6 +473,9 @@ def print_top_results(items, n=5):
print(f"Top {n} DBA results (title — price — location):\n")
for it in items[:n]:
loc = it.get('location') or ''
dist = it.get('distance_km')
if dist is not None:
loc = f"{loc} ({dist:.0f} km)"
print(f"- {it.get('title','')}")
print(f" price: {it.get('price','')}\n location: {loc}\n url: {it.get('url','')}\n")
@@ -452,15 +487,19 @@ def print_comparison_table(dba_items, pr_items, n=5):
for i in range(maxrows):
left = dba_items[i] if i < len(dba_items) else None
right = pr_items[i] if i < len(pr_items) else None
loc = left.get('location','') if left else ''
dist = left.get('distance_km') if left else None
if dist is not None:
loc = f"{loc} ({dist:.0f} km)"
rows.append((
str(i+1),
_short(left.get('title','')) if left else '',
left.get('price','') if left else '',
left.get('location','') if left else '',
loc,
_short(right.get('title','')) if right else '',
right.get('price','') if right else '',
))
widths = [3, 50, 12, 12, 50, 12]
widths = [3, 50, 12, 18, 50, 12]
hdr = ('#', 'DBA title', 'DBA price', 'DBA loc', 'PriceRunner title', 'PR price')
sep = ' | '
def fmt(row):
@@ -490,6 +529,9 @@ def print_comparison_markdown(dba_items, pr_items, n=10):
title_md = title
price = it.get('price','')
loc = it.get('location','')
dist = it.get('distance_km')
if dist is not None:
loc = f"{loc} ({dist:.0f} km)"
parts = [title_md, price]
if loc:
parts.append(loc)
@@ -517,6 +559,9 @@ def print_comparison_grid(dba_items, pr_items, n=10):
title = it.get('title','')
price = it.get('price','')
loc = it.get('location','')
dist = it.get('distance_km')
if dist is not None:
loc = f"{loc} ({dist:.0f} km)"
url = it.get('url','') or ''
if url and len(url) > 80:
url = url[:77] + '...'
@@ -588,6 +633,8 @@ def main():
help='Filter out items below this price (e.g. 500 or "3.000")')
parser.add_argument('--max-price', type=str, default=None,
help='Filter out items above this price')
parser.add_argument('--from', dest='origin', type=str, default=None,
help='Your location for distance calculation (e.g. "Farum")')
args = parser.parse_args()
query = ' '.join(args.query)
print(f"Searching for: {query}\n")
@@ -595,9 +642,9 @@ def main():
# ── Fetch DBA ──
try:
if args.engine == 'requests':
dba = search_dba_requests(query, max_results=args.max)
dba = search_dba_requests(query, max_results=args.max, origin=args.origin)
else:
dba = search_dba_selenium(query, max_results=args.max)
dba = search_dba_selenium(query, max_results=args.max, origin=args.origin)
except ScraperError as e:
print("Error while scraping DBA:", e)
sys.exit(1)
@@ -692,7 +739,11 @@ def main():
dba_sorted = sort_items_by_price(dba)
print(f"DBA — top {min(len(dba_sorted), args.max)} by price:")
for it in dba_sorted[:args.max]:
print(f"- {it.get('title','')} — {it.get('price','')} — {it.get('location','')} — {it.get('url','')}")
loc = it.get('location','')
dist = it.get('distance_km')
if dist is not None:
loc = f"{loc} ({dist:.0f} km)"
print(f"- {it.get('title','')} — {it.get('price','')} — {loc} — {it.get('url','')}")
if __name__ == '__main__':

1
requirements.txt
View File

@@ -1,5 +1,6 @@
requests
beautifulsoup4
lxml
geopy
webdriver-manager
selenium