From a56a911bee6f8cec378bc8a713927d4008418ec7 Mon Sep 17 00:00:00 2001 From: alexpolo1 Date: Thu, 20 Aug 2026 14:56:29 +0200 Subject: [PATCH] feat: import reference task prices from haandvaerkpriser.dk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support ticket #396994 (Jannick): pull prices from haandvaerkpriser.dk for tasks we don't already price, as a starting reference. robots.txt allows crawling /alle-priser/ and no terms of use restrict reuse (checked before building this). Each imported task becomes a verified 'component' Smart Package with a single task carrying the price, filtered to only the 8 trades Jannick asked for (Maler, Gulv, VVS & Blikkenslager, Tømrer & Snedker, Murer, Tagrenovering, Isolering & Energi, Nedrivning & Affald - the site has 16 categories total). This lands in the same place as today's earlier smart-package fix, so these are immediately selectable via the "Enkelte opgaver" picker and visible in the admin Smart Packages list, unlike the existing price_database table which is dead code (0 rows, its only consumer queries a column that no longer exists, and its route isn't mounted). Re-running the import is idempotent: a task already imported from this source gets its price averaged with the new pull instead of duplicating, per Jannick's request ("har vi en pris i forvejen kan den lave et gennemsnit"). Cross-source matching against Jannick's own manually-curated components is intentionally out of scope - automating that risks merging unrelated tasks. New endpoints (admin-only): GET /api/smart-packages/haandvaerkpriser-preview (dry run, no writes) and POST /api/smart-packages/haandvaerkpriser-import. Co-Authored-By: Claude Sonnet 5 --- .../haandvaerkPriserImportService.test.js | 204 ++++++++++++++++++ backend/src/routes/smartPackagesRoutes.js | 24 +++ .../services/haandvaerkPriserImportService.js | Bin 0 -> 9223 bytes 3 files changed, 228 insertions(+) create mode 100644 backend/__tests__/haandvaerkPriserImportService.test.js create mode 100644 backend/src/services/haandvaerkPriserImportService.js diff --git a/backend/__tests__/haandvaerkPriserImportService.test.js b/backend/__tests__/haandvaerkPriserImportService.test.js new file mode 100644 index 0000000..b84f65c --- /dev/null +++ b/backend/__tests__/haandvaerkPriserImportService.test.js @@ -0,0 +1,204 @@ +const HaandvaerkPriserImportService = require('../src/services/haandvaerkPriserImportService'); +const { + parseHaandvaerkPriserHtml, + parsePrice, + parsePriceRange, + normalizeUnit, + normalizeName +} = HaandvaerkPriserImportService; + +const section = (category, rows) => ` +
+
+

${category}

+
+
+ + + + ${rows.map((row) => ` + + + + + + + `).join('')} + +
OpgaveGns. pris inkl. momsPrisintervalEnhed
${row.name}${row.avg} kr${row.low} – ${row.high} kr${row.unit}
+
+
+`; + +const fixtureHtml = ` + ${section('Maler', [ + { name: 'Maling af lejlighed (vægge og lofter)', avg: '110', low: '60', high: '120', unit: 'pr. m² vægflade', href: '/maler/maling/' } + ])} + ${section('Tømrer & Snedker', [ + { name: 'Udskiftning af vinduer', avg: '10.000', low: '5.000', high: '15.000', unit: 'pr. stk' } + ])} + ${section('Elektriker', [ + { name: 'Skift af gruppetavle', avg: '5.000', low: '3.000', high: '7.000', unit: 'fast pris' } + ])} +`; + +describe('parsePrice', () => { + test.each([ + ['110\n kr', 110], + ['10.000 kr', 10000], + ['', 0], + [null, 0] + ])('parses %p as %p', (input, expected) => { + expect(parsePrice(input)).toBe(expected); + }); +}); + +describe('parsePriceRange', () => { + test('splits a Danish en-dash range', () => { + expect(parsePriceRange('60 – 120 kr')).toEqual({ low: 60, high: 120 }); + }); + + test('returns nulls for malformed ranges', () => { + expect(parsePriceRange('n/a')).toEqual({ low: null, high: null }); + }); +}); + +describe('normalizeUnit', () => { + test.each([ + ['pr. m² vægflade', { unit: 'm²', timeUnit: 'per_sqm' }], + ['pr. løb. m', { unit: 'løbende m', timeUnit: 'per_meter' }], + ['pr. stk', { unit: 'stk', timeUnit: 'per_piece' }], + ['pr. time', { unit: 'time', timeUnit: 'per_hour' }] + ])('maps %p', (input, expected) => { + expect(normalizeUnit(input)).toEqual(expected); + }); + + test('falls back to a fixed price for long-tail units', () => { + expect(normalizeUnit('pr. lejl.')).toEqual({ unit: 'pr. lejl.', timeUnit: 'per_project' }); + expect(normalizeUnit('')).toEqual({ unit: 'fast pris', timeUnit: 'per_project' }); + }); +}); + +describe('normalizeName', () => { + test('strips accents and punctuation for stable matching', () => { + expect(normalizeName('Udskiftning af vinduer (2-lag)')).toBe('udskiftning af vinduer 2 lag'); + expect(normalizeName('Tømrer & Snedker')).toBe('t mrer snedker'); + }); +}); + +describe('parseHaandvaerkPriserHtml', () => { + test('only keeps rows from allowed categories, decoding HTML entities in category names', () => { + const rows = parseHaandvaerkPriserHtml(fixtureHtml); + expect(rows).toHaveLength(2); + expect(rows.map((row) => row.category)).toEqual(['Maler', 'Tømrer & Snedker']); + expect(rows.find((row) => row.category === 'Elektriker')).toBeUndefined(); + }); + + test('extracts price, range, unit and an absolute detail URL per row', () => { + const rows = parseHaandvaerkPriserHtml(fixtureHtml); + expect(rows[0]).toMatchObject({ + name: 'Maling af lejlighed (vægge og lofter)', + avgPrice: 110, + priceRangeLow: 60, + priceRangeHigh: 120, + unit: 'm²', + timeUnit: 'per_sqm', + detailUrl: 'https://haandvaerkpriser.dk/maler/maling/' + }); + }); + + test('respects a custom allow-list', () => { + const rows = parseHaandvaerkPriserHtml(fixtureHtml, { categories: new Set(['Elektriker']) }); + expect(rows).toHaveLength(1); + expect(rows[0].name).toBe('Skift af gruppetavle'); + }); +}); + +describe('HaandvaerkPriserImportService.import', () => { + const buildConnection = (existingRows) => { + const executedInserts = []; + const executedUpdates = []; + const execute = jest.fn(async (sql, params) => { + if (sql.includes('SELECT id, name, category, unit_price')) return [existingRows]; + if (sql.includes('INSERT INTO material_packages')) { + executedInserts.push({ sql, params }); + return [{ insertId: 900 + executedInserts.length }]; + } + if (sql.includes('INSERT INTO smart_package_tasks')) { + executedInserts.push({ sql, params }); + return [{}]; + } + if (sql.includes('UPDATE material_packages') || sql.includes('UPDATE smart_package_tasks')) { + executedUpdates.push({ sql, params }); + return [{}]; + } + throw new Error(`Unexpected SQL: ${sql}`); + }); + const connection = { execute, beginTransaction: jest.fn(), commit: jest.fn(), rollback: jest.fn(), release: jest.fn() }; + return { connection, executedInserts, executedUpdates }; + }; + + test('inserts a new verified component package with one task carrying the price', async () => { + const { connection, executedInserts } = buildConnection([]); + const service = new HaandvaerkPriserImportService({ pool: { getConnection: async () => connection } }); + service.fetchRows = jest.fn().mockResolvedValue([ + { category: 'Maler', name: 'Maling af lejlighed', avgPrice: 110, priceRangeLow: 60, priceRangeHigh: 120, unit: 'm²', timeUnit: 'per_sqm', rawUnitText: 'pr. m²', detailUrl: 'https://haandvaerkpriser.dk/maler/x/' } + ]); + + const result = await service.import(); + + expect(result).toEqual({ totalRows: 1, insertedPackages: 1, updatedPackages: 0 }); + expect(connection.commit).toHaveBeenCalled(); + const packageInsert = executedInserts.find((call) => call.sql.includes('INSERT INTO material_packages')); + expect(packageInsert.params).toEqual(expect.arrayContaining(['Maling af lejlighed', expect.any(String), 'Maler'])); + expect(packageInsert.sql).toContain("'component'"); + expect(packageInsert.sql).toContain("'verified'"); + const taskInsert = executedInserts.find((call) => call.sql.includes('INSERT INTO smart_package_tasks')); + expect(taskInsert.params).toEqual([901, 'Maling af lejlighed', expect.any(String), 110, 'per_sqm', expect.any(String)]); + }); + + test('averages the price with the existing package on a repeat import instead of duplicating', async () => { + const { connection, executedUpdates } = buildConnection([ + { id: 42, name: 'Maling af lejlighed', category: 'Maler', unit_price: '90.00' } + ]); + const service = new HaandvaerkPriserImportService({ pool: { getConnection: async () => connection } }); + service.fetchRows = jest.fn().mockResolvedValue([ + { category: 'Maler', name: 'Maling af lejlighed', avgPrice: 110, priceRangeLow: 60, priceRangeHigh: 120, unit: 'm²', timeUnit: 'per_sqm', rawUnitText: 'pr. m²', detailUrl: 'https://haandvaerkpriser.dk/maler/x/' } + ]); + + const result = await service.import(); + + expect(result).toEqual({ totalRows: 1, insertedPackages: 0, updatedPackages: 1 }); + const packageUpdate = executedUpdates.find((call) => call.sql.includes('UPDATE material_packages')); + // (90 + 110) / 2 = 100 + expect(packageUpdate.params).toEqual(expect.arrayContaining([100])); + }); + + test('rolls back and rethrows on a database error', async () => { + const { connection } = buildConnection([]); + connection.execute = jest.fn().mockRejectedValue(new Error('db down')); + const service = new HaandvaerkPriserImportService({ pool: { getConnection: async () => connection } }); + service.fetchRows = jest.fn().mockResolvedValue([ + { category: 'Maler', name: 'X', avgPrice: 100, priceRangeLow: 1, priceRangeHigh: 2, unit: 'stk', timeUnit: 'per_piece', rawUnitText: 'pr. stk', detailUrl: 'https://x' } + ]); + + await expect(service.import()).rejects.toThrow('db down'); + expect(connection.rollback).toHaveBeenCalled(); + }); +}); + +describe('HaandvaerkPriserImportService.preview', () => { + test('summarizes counts per category without writing anything', async () => { + const service = new HaandvaerkPriserImportService({ pool: {} }); + service.fetchRows = jest.fn().mockResolvedValue([ + { category: 'Maler', name: 'A' }, + { category: 'Maler', name: 'B' }, + { category: 'Murer', name: 'C' } + ]); + + const result = await service.preview(); + expect(result.totalTasks).toBe(3); + expect(result.byCategory).toEqual({ Maler: 2, Murer: 1 }); + expect(result.sample).toHaveLength(3); + }); +}); diff --git a/backend/src/routes/smartPackagesRoutes.js b/backend/src/routes/smartPackagesRoutes.js index 824c1fe..06dacc7 100644 --- a/backend/src/routes/smartPackagesRoutes.js +++ b/backend/src/routes/smartPackagesRoutes.js @@ -12,6 +12,7 @@ const AiFeatureFlagService = require('../services/aiFeatureFlagService'); const { verifyToken } = require('../middleware/auth'); const aiValidationJobs = require('../services/aiValidationJobService'); const { SmartPackageIntegrityService } = require('../services/smartPackageIntegrityService'); +const HaandvaerkPriserImportService = require('../services/haandvaerkPriserImportService'); const excelUploadDir = path.join(__dirname, '../../uploads/smart-packages/excel'); const excelMetadataPath = path.join(excelUploadDir, '.import-metadata.json'); @@ -268,6 +269,29 @@ router.get('/integrity-report', async (req, res) => { } }); +// Referencepriser fra haandvaerkpriser.dk (support ticket #396994). Preview +// henter og parser siden uden at skrive noget - bruges til at vise en admin +// hvad en import ville medføre, før den faktisk køres. +router.get('/haandvaerkpriser-preview', verifyToken, async (req, res) => { + try { + const preview = await new HaandvaerkPriserImportService(global.databaseService).preview(); + res.json({ success: true, ...preview }); + } catch (error) { + console.error('Error previewing haandvaerkpriser.dk import:', error); + res.status(500).json({ success: false, error: 'Kunne ikke hente forhåndsvisning: ' + error.message }); + } +}); + +router.post('/haandvaerkpriser-import', verifyToken, async (req, res) => { + try { + const result = await new HaandvaerkPriserImportService(global.databaseService).import(); + res.json({ success: true, ...result }); + } catch (error) { + console.error('Error importing from haandvaerkpriser.dk:', error); + res.status(500).json({ success: false, error: 'Import fejlede: ' + error.message }); + } +}); + // Pakker som integritetskontrollen har arkiveret (blokeret + inaktiv) og som // venter på manuel materialekobling, med forslag til match fra materialedatabasen. router.get('/review-queue', verifyToken, async (req, res) => { diff --git a/backend/src/services/haandvaerkPriserImportService.js b/backend/src/services/haandvaerkPriserImportService.js new file mode 100644 index 0000000000000000000000000000000000000000..41955b9b347a221b4b0bdf936efb5571dd985161 GIT binary patch literal 9223 zcmbVS+j84R65VHgMaQc$K!*e=XOmjXt98jX9V?b)moCYJqEI7ZNFoFQW&p@m6jtp& z>^CG;`?^ngUj4~_$)27I0D`2liOZINx%Bk(oIc$>@GwoXoW#c{&1jF5{Go`H^gZ3+ zwVrMbb^mZ8rHaz}2-{SRiPP~|s``*1=oWbtXI<09L#@1%cW1{h`h&CMLyY$(c|Ol} zyWNQpNq8-!n$A^}Nfm_Cu83pVF|S?Esbi-f?Z55o`K$9VbgSE?H$_6Bq>&iYS0YKG z;glvq(UlByN@wI}#eANsoO1NYoc{dTlP{h;`AdsBltrOT^x&N@SkhGFa-6CNJB&!g zd7EbFzmD)Xrf52q6wO$aR9NmK^8)`*o(U z(nC5)WQdhb^V(Y(Mc0Yx*dL8V9C}+9 z)d9|h%H(?$4JDNJbNMlE(ca(b7P=b5gwZ(48B!-OLo(Jw=%=5^^ICzD^H>a}-#!2C z+wh`0ZgcBXOjR;36i&>{eMU#c>`E%%ytinVdQZ0&C5S2=jzu!wqzFcB(j%M4QSN*H z{l`C{)J)8Mp9kxWRI*K*RxcWne#?SbCgXfUpX}|?W8RfsB*@a4^!;lQ7g7uP#ONPH z*{djta%sC-Ep4ZAySEglA9g7z;}@RErPfw(FDB{WjPg4zc@AIw!iv;X{?uJ3*sk75zZ2W@&V65s-MHuA5~ggzzX zOtv1hSZX?lpTzJd_Lnd-;ycqc$(5MqbTWhS=)C|g6+8s6l5o8%RWK`H4zvgZIs2dQ3&3>+JZ&dWCj_>3%`9z@A7E<&049uCKA@$6>% zV{a8e1+>{EJ*N%R%(SLxm`gRtewcX`me#Bk-~D>EStj?gp=jo$Y!JFRDquiOy_K*r2x?(s^jtWx2b)pCy`?+-sc@rXLe4PNK7m|by#-)*|3p^MKPbJm2`1)6Uc4UgSMqs?Q2 zI2ECmrm5ZpF_nH`apxDX4$-#nP2hZi6vK({55T`tgotgWP#62GE;pFE?Ylt%`?24$ zcw%ZNVrTrM(ma@m%y*OFG2q_z_3&Vf?4{SjK3BKvW{B&t zP907SVb*8v+&@!spkFiX@%_`-$JfmI?EYB@bRvphD>>3OSl*s7vVajqo+}^&!wg?@ z9Jgm?Pna!KohBcA(-K%fqgLkIW2E>@J#Ofpoq03sZ z6l|bwh_GT!J-H+tj&{DYB4%pzB7uQ_1oI-B_?-BeKMuc~wwgb@{p!u_p=v?GF_c<6 z1jk)YhSv4H)`=0PM^!ItRK2;lSG8CY=cC`%M9)uEw>jW574LFVCHL%qLM6gjYE1#sHH)LWsz2k|B*`KAapw1B5ekUY6x#Hb<4mv?4wTWH`F` z(~ewi4jgOu$Al(Twp%u!JUIg+clJU4xradcHvKV*W6}LQ=#f86XLFH9pslBLisY>b z#}O)!L_^)|x4RScTDtBqfbsT+LpjeohrnuqWRGR)WT_pBr$6lVf+wEUW#JB!&ps`1 z$~&L;dOh9i&cT{~V-)d`BkKJJY(9N>3qCR(4hgaArMl^;BH0DsBQmCA<5MP?Hf3No zMX`h!naez&=bZ6oIzbgVl1&(PaxFO4v09I8oyc_~m>fZHboZ-5y34;zh~)PS0T6?` z_-PJruHTl)K`28mHk8`&Y6#OhAy%cih)+c}1;n(5rr2^PT`lce zGBKOS(hkmIc#hv|)90KXp?+dTme13GPAB;jAQ5#9Co+WUjq?$%jz^FI*pSEIO%sN7 zC%>7?dX2+J$16OdyMHBCMjHo++XXJ0l&+&7&;NcD01Qj_3jv`ZM3xUEaTQ-xODNKe zOf;g#iX+rc?q-3+i1Cp~nnoyv+E^2SMSrGYqtmGz)eJ_=G7zI3Z{&@|3}^#(xh_g> zq^a^EU6f&?009B4DNvT9Uc#8LfXC)d6q-AZKMF$7O@b^rN|TamEO!O-G>w^YUX(+8 zOQQzc)zsihj-%uhaV!&B77yf2B8RytIms~Fx%?PqIa0J^_z44dMVS03hXpEgEr5kw zp7am$V+CcEw$$Ps|sDrB*wwLKIG!z@%wx6+u zD8Sc@Se4R|_G%Y`Sc2)C>RtXG&7B5Lg0l>?bC241I+Q_tY2b3gXvBEDE^o5jw9=47 z&tUX?5~#F5R;_P{GWiO{LS7z3z-frVg)Z1N0kVNzs?(w_JvMa2+a+WT_3q|2xKhnH z>pe@kdP*biWnre=%5_xA2jqT?mi?X8>G+>rr zO3&!wL#Rb!edc3 zTJXj1B5;0VEPAdt`#KqqB226UuV zHvrMu`$T>iB9^b_=9Y<{8D@he@T9>vU__gD#?*kXk!;gWMHfa^ z1%5L@pLBf&gl0J|_Js;9dGbOHjWi$!C}SfwQ+*&+jaoOXvSZYYZw3m3^|B3dMhWD~ zQ8q{nl%a-cHOa@oT)pT$TS#D8#K1i|diRarYAT4dATxo>Jt4_iTGdc~6tX_Gj>|Tk z*Ip4})g_fm&R(_mDFGqE9)mLYXfZ%>7>Yl z`l80T_uzPGA&AS0Q|BZpHpp=r>|2(ba!2Ihe|m0F3x3ZAdns@lXw zO{b+zG$Y-oSgfL$1-|wm-@mQRwwVahv?9*$%|Vvk`QYfJe|$;@N2l-B_;i)97*m^6 zrRLY|THey|pe65FMw89Hn~vBfflOw=wI|Y|((brft`kO6lruhO&TXoAcK|xbi)^Wh zowRDvp4b#;+@M_Cyz)=OD$czfS2bCuDXvKoZoZ+1o4o8Qq3)_-S<>8aHN+^_*gbXE=2*u=w4K<7&MXI$7Fw z(|zvERd$f1pep-+#+3Qb@{v^l(t*3qjZIB{<9%rtO=7F17!8)IO_9M)pU^B_BTPk zL@z%y!CBhJnDq}pvw5s_FjNn0orUw#~;vRk_QAa{>aUL@Xb7}@^q zMa-25E=4#(3@6Asl}d5_?}FCK-c=goo9l2|?LMEt^!Un(c?Q}G!hJZt`7RaUw${fd zs#x*|Pis8>qP^IHsb^_e#4?Z{`TtFDB3{|?y|$yt%TLU0=+nmdVB>`zByH?lF9pl# z|Hx2Xmftf@-&DIb^i18dGNUZG&rCAC^uBrYk@=ibj^_83(Q7_%t^4Y=GOxOA!v?CC Hr(6F67`9M^ literal 0 HcmV?d00001