From 243bc9b6f5eb7ecb187289300c6c8ca4cdd62d41 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 28 Aug 2026 16:04:11 +0200 Subject: [PATCH] Add --from flag: travel distance from your location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Geocodes listing locations via Photon (Komoot) API - no rate limits - Shows distance in km next to location in all output formats - Uses geodesic distance (geopy) for accurate km calculation - Caches geocoded locations to avoid repeated API calls - Works with --top, --compare, --verdict, and default output - Example: --from "Farum" shows 'København V (20 km)', 'Korsør (94 km)' - Added geopy to requirements.txt --- README.md | 4 ++ dba_pricerunner_scraper.py | 105 +++++++++++++++++++++++++++---------- requirements.txt | 1 + 3 files changed, 83 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index d82db99..0e39c43 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ Output: | `--format` | `text` (default), `markdown`, or `grid` | | `--top N` | Show top N results | | `--min-price` / `--max-price` | Filter by price (e.g. `--min-price 2000`) | +| `--from` | Your location for distance calculation (e.g. `--from Farum`) | | `--json` | Output raw JSON | | `--engine selenium` | Use Selenium instead of requests (for JS-heavy pages) | @@ -62,6 +63,9 @@ python3 dba_pricerunner_scraper.py "samsung galaxy s24" --pricerunner --compare # Filter out accessories: only look at items 2000–5000 kr python3 dba_pricerunner_scraper.py "playstation 5 konsol" --pricerunner --compare --verdict --min-price 2000 --max-price 5000 +# Show distances from your location (e.g. Farum) +python3 dba_pricerunner_scraper.py "iphone 15" --from "Farum" --pricerunner --compare --verdict + # ASCII grid for terminal python3 dba_pricerunner_scraper.py "xbox series x konsol" --pricerunner --compare --verdict --format grid diff --git a/dba_pricerunner_scraper.py b/dba_pricerunner_scraper.py index 9184265..60d893e 100755 --- a/dba_pricerunner_scraper.py +++ b/dba_pricerunner_scraper.py @@ -20,6 +20,40 @@ import requests from bs4 import BeautifulSoup from urllib.parse import quote_plus, urljoin +try: + from geopy.distance import geodesic + import requests as _requests + _GEO_CACHE = {} + def _geocode(name): + if not name: + return None + if name in _GEO_CACHE: + return _GEO_CACHE[name] + try: + r = _requests.get( + 'https://photon.komoot.io/api/', + params={'q': f'{name}, Danmark'}, + headers={'User-Agent': 'dba_pricerunner/1.0'}, + timeout=10 + ) + if r.status_code == 200: + data = r.json() + features = data.get('features', []) + if features: + coords = features[0]['geometry']['coordinates'] + result = (coords[1], coords[0]) # [lon, lat] -> (lat, lon) + else: + result = None + else: + result = None + except Exception: + result = None + _GEO_CACHE[name] = result + return result +except ImportError: + def _geocode(name): + return None + try: from selenium import webdriver from selenium.webdriver.chrome.service import Service as ChromeService @@ -118,26 +152,10 @@ def extract_price_string(text): # DBA scraping # ────────────────────────────────────────────────────────────── -def _parse_dba_article(el): +def _parse_dba_article(el, origin_coords=None): """Extract title, price, url, location from a DBA
element. - DBA's current markup: - + If origin_coords is provided, also computes travel distance from origin. """ # Title: h2 text directly (the h2 has NO child in current markup) h2 = el.select_one('h2') @@ -175,6 +193,13 @@ def _parse_dba_article(el): if sp and sp.get_text(strip=True): location = sp.get_text(strip=True) + # Distance from origin (if provided) + distance_km = None + if origin_coords and location: + coords = _geocode(location) + if coords: + distance_km = geodesic(origin_coords, coords).km + price_num = normalize_price(price) return { "site": "dba", @@ -183,11 +208,15 @@ def _parse_dba_article(el): "price_num": price_num, "url": link, "location": location, + "distance_km": distance_km, } -def search_dba_requests(query, max_results=10): +def search_dba_requests(query, max_results=10, origin=None): """Search dba.dk (requests) and return list of dicts with title, price, url, location.""" + origin_coords = None + if origin: + origin_coords = _geocode(origin) q = quote_plus(query) url = f"https://www.dba.dk/recommerce/forsale/search?q={q}" r = requests.get(url, headers=HEADERS, timeout=15) @@ -200,17 +229,20 @@ def search_dba_requests(query, max_results=10): # skip skeleton/placeholder articles (no text content) if not el.get_text(strip=True): continue - item = _parse_dba_article(el) + item = _parse_dba_article(el, origin_coords) if item['title']: items.append(item) time.sleep(1) return items -def search_dba_selenium(query, max_results=10, headless=True): +def search_dba_selenium(query, max_results=10, headless=True, origin=None): """Render DBA with Selenium and extract the same fields as requests path.""" if webdriver is None: raise ScraperError('Selenium or webdriver-manager not installed') + origin_coords = None + if origin: + origin_coords = _geocode(origin) opts = ChromeOptions() if headless: opts.add_argument('--headless=new') @@ -228,7 +260,7 @@ def search_dba_selenium(query, max_results=10, headless=True): for el in results[:max_results]: if not el.get_text(strip=True): continue - item = _parse_dba_article(el) + item = _parse_dba_article(el, origin_coords) if item['title']: items.append(item) time.sleep(1) @@ -441,6 +473,9 @@ def print_top_results(items, n=5): print(f"Top {n} DBA results (title — price — location):\n") for it in items[:n]: loc = it.get('location') or '' + dist = it.get('distance_km') + if dist is not None: + loc = f"{loc} ({dist:.0f} km)" print(f"- {it.get('title','')}") print(f" price: {it.get('price','')}\n location: {loc}\n url: {it.get('url','')}\n") @@ -452,15 +487,19 @@ def print_comparison_table(dba_items, pr_items, n=5): for i in range(maxrows): left = dba_items[i] if i < len(dba_items) else None right = pr_items[i] if i < len(pr_items) else None + loc = left.get('location','') if left else '' + dist = left.get('distance_km') if left else None + if dist is not None: + loc = f"{loc} ({dist:.0f} km)" rows.append(( str(i+1), _short(left.get('title','')) if left else '', left.get('price','') if left else '', - left.get('location','') if left else '', + loc, _short(right.get('title','')) if right else '', right.get('price','') if right else '', )) - widths = [3, 50, 12, 12, 50, 12] + widths = [3, 50, 12, 18, 50, 12] hdr = ('#', 'DBA title', 'DBA price', 'DBA loc', 'PriceRunner title', 'PR price') sep = ' | ' def fmt(row): @@ -490,6 +529,9 @@ def print_comparison_markdown(dba_items, pr_items, n=10): title_md = title price = it.get('price','') loc = it.get('location','') + dist = it.get('distance_km') + if dist is not None: + loc = f"{loc} ({dist:.0f} km)" parts = [title_md, price] if loc: parts.append(loc) @@ -517,6 +559,9 @@ def print_comparison_grid(dba_items, pr_items, n=10): title = it.get('title','') price = it.get('price','') loc = it.get('location','') + dist = it.get('distance_km') + if dist is not None: + loc = f"{loc} ({dist:.0f} km)" url = it.get('url','') or '' if url and len(url) > 80: url = url[:77] + '...' @@ -588,6 +633,8 @@ def main(): help='Filter out items below this price (e.g. 500 or "3.000")') parser.add_argument('--max-price', type=str, default=None, help='Filter out items above this price') + parser.add_argument('--from', dest='origin', type=str, default=None, + help='Your location for distance calculation (e.g. "Farum")') args = parser.parse_args() query = ' '.join(args.query) print(f"Searching for: {query}\n") @@ -595,9 +642,9 @@ def main(): # ── Fetch DBA ── try: if args.engine == 'requests': - dba = search_dba_requests(query, max_results=args.max) + dba = search_dba_requests(query, max_results=args.max, origin=args.origin) else: - dba = search_dba_selenium(query, max_results=args.max) + dba = search_dba_selenium(query, max_results=args.max, origin=args.origin) except ScraperError as e: print("Error while scraping DBA:", e) sys.exit(1) @@ -692,7 +739,11 @@ def main(): dba_sorted = sort_items_by_price(dba) print(f"DBA — top {min(len(dba_sorted), args.max)} by price:") for it in dba_sorted[:args.max]: - print(f"- {it.get('title','')} — {it.get('price','')} — {it.get('location','')} — {it.get('url','')}") + loc = it.get('location','') + dist = it.get('distance_km') + if dist is not None: + loc = f"{loc} ({dist:.0f} km)" + print(f"- {it.get('title','')} — {it.get('price','')} — {loc} — {it.get('url','')}") if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index a3d71a3..197554f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ requests beautifulsoup4 lxml +geopy webdriver-manager selenium