283 lines
10 KiB
JavaScript
283 lines
10 KiB
JavaScript
const path = require('path');
|
|
const fs = require('fs/promises');
|
|
const mysql = require('mysql2/promise');
|
|
|
|
require('dotenv').config({
|
|
path: path.join(__dirname, '..', 'backend', '.env')
|
|
});
|
|
|
|
const REPORT_PATH = path.join(
|
|
__dirname,
|
|
'..',
|
|
'docs',
|
|
'status-reports',
|
|
'ORDRESTYRING_FULL_DATA_QUALITY_2026-04-06.md'
|
|
);
|
|
|
|
async function main() {
|
|
if (!process.env.DB_PASSWORD) {
|
|
throw new Error('DB_PASSWORD is required');
|
|
}
|
|
|
|
const conn = await mysql.createConnection({
|
|
host: process.env.DB_HOST || '127.0.0.1',
|
|
port: parseInt(process.env.DB_PORT, 10) || 3306,
|
|
user: process.env.DB_USER || 'tilbudgivern_service',
|
|
password: process.env.DB_PASSWORD,
|
|
database: process.env.DB_NAME || 'tilbudgivern'
|
|
});
|
|
|
|
try {
|
|
await conn.execute(
|
|
`
|
|
CREATE TABLE IF NOT EXISTS ordrestyring_data_quality_issues (
|
|
id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
|
issue_type VARCHAR(64) NOT NULL,
|
|
severity VARCHAR(16) NOT NULL,
|
|
entity_type VARCHAR(32) NOT NULL,
|
|
entity_key VARCHAR(255) NULL,
|
|
issue_details_json JSON NULL,
|
|
detected_at TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP,
|
|
resolved_at DATETIME NULL,
|
|
INDEX idx_quality_issue_type (issue_type),
|
|
INDEX idx_quality_entity (entity_type, entity_key),
|
|
INDEX idx_quality_detected (detected_at)
|
|
)
|
|
`
|
|
);
|
|
|
|
await conn.execute('DELETE FROM ordrestyring_data_quality_issues WHERE resolved_at IS NULL');
|
|
|
|
const [[latestCaseTableStats]] = await conn.execute(
|
|
`
|
|
SELECT
|
|
COUNT(*) AS table_count
|
|
FROM INFORMATION_SCHEMA.TABLES
|
|
WHERE TABLE_SCHEMA = DATABASE()
|
|
AND TABLE_NAME = 'ordrestyring_case_latest'
|
|
`
|
|
);
|
|
|
|
const hasCaseLatestTable = Number(latestCaseTableStats.table_count || 0) > 0;
|
|
let latestCaseRows = 0;
|
|
if (hasCaseLatestTable) {
|
|
const [[latestCaseRowsStats]] = await conn.execute(
|
|
`
|
|
SELECT COUNT(*) AS total_rows
|
|
FROM ordrestyring_case_latest
|
|
`
|
|
);
|
|
latestCaseRows = Number(latestCaseRowsStats.total_rows || 0);
|
|
}
|
|
|
|
const [[caseStats]] = await conn.execute(
|
|
`
|
|
SELECT
|
|
COUNT(*) AS total_case_rows,
|
|
COUNT(DISTINCT case_number) AS distinct_case_numbers
|
|
FROM ordrestyring_local.cases
|
|
WHERE case_number IS NOT NULL AND case_number <> ''
|
|
`
|
|
);
|
|
|
|
const [[hourStats]] = await conn.execute(
|
|
`
|
|
SELECT
|
|
COUNT(*) AS total_hours,
|
|
SUM(CASE WHEN new_case_number IS NULL OR new_case_number = '' THEN 1 ELSE 0 END) AS hours_without_case,
|
|
SUM(CASE WHEN remark IS NULL OR TRIM(remark) = '' THEN 1 ELSE 0 END) AS hours_without_remark
|
|
FROM ordrestyring_local.hours
|
|
`
|
|
);
|
|
|
|
const [[hourLinkStats]] = await conn.execute(
|
|
`
|
|
SELECT
|
|
SUM(
|
|
CASE
|
|
WHEN (h.new_case_number IS NULL OR h.new_case_number = '')
|
|
AND h.hour_type IN (
|
|
SELECT DISTINCT hour_type
|
|
FROM ordrestyring_local.hours
|
|
WHERE new_case_number IS NOT NULL
|
|
AND new_case_number <> ''
|
|
AND hour_type IS NOT NULL
|
|
)
|
|
THEN 1
|
|
ELSE 0
|
|
END
|
|
) AS hours_without_case_case_related
|
|
FROM ordrestyring_local.hours h
|
|
`
|
|
);
|
|
|
|
const [[materialStats]] = await conn.execute(
|
|
`
|
|
SELECT
|
|
COUNT(*) AS total_materials,
|
|
SUM(CASE WHEN case_number IS NULL OR case_number = '' THEN 1 ELSE 0 END) AS materials_without_case,
|
|
SUM(CASE WHEN product_text IS NULL OR TRIM(product_text) = '' THEN 1 ELSE 0 END) AS materials_without_text,
|
|
SUM(CASE WHEN CAST(COALESCE(NULLIF(sales_price, ''), '0') AS DECIMAL(14,2)) = 0 THEN 1 ELSE 0 END) AS materials_zero_price
|
|
FROM ordrestyring_local.case_materials
|
|
`
|
|
);
|
|
|
|
const [[normMatStats]] = await conn.execute(
|
|
`
|
|
SELECT
|
|
COUNT(*) AS rows_total,
|
|
SUM(CASE WHEN normalized_unit = 'm2' THEN 1 ELSE 0 END) AS rows_m2,
|
|
SUM(CASE WHEN normalized_unit = 'lbm' THEN 1 ELSE 0 END) AS rows_lbm,
|
|
SUM(CASE WHEN normalized_unit = 'stk' THEN 1 ELSE 0 END) AS rows_stk
|
|
FROM ordrestyring_materials_normalized
|
|
`
|
|
);
|
|
|
|
const [[normHourStats]] = await conn.execute(
|
|
`
|
|
SELECT
|
|
COUNT(*) AS rows_total,
|
|
SUM(CASE WHEN duration_hours > 0 THEN 1 ELSE 0 END) AS rows_with_duration,
|
|
SUM(CASE WHEN remark IS NOT NULL AND TRIM(remark) <> '' THEN 1 ELSE 0 END) AS rows_with_remark
|
|
FROM ordrestyring_hours_normalized
|
|
`
|
|
);
|
|
|
|
const duplicateRatio = caseStats.distinct_case_numbers > 0
|
|
? (caseStats.total_case_rows / caseStats.distinct_case_numbers)
|
|
: 0;
|
|
|
|
const issues = [];
|
|
if (latestCaseRows === 0) {
|
|
issues.push({
|
|
issueType: 'CASE_DUPLICATION',
|
|
severity: duplicateRatio > 20 ? 'high' : (duplicateRatio > 5 ? 'medium' : 'low'),
|
|
entityType: 'cases',
|
|
entityKey: 'global',
|
|
details: {
|
|
totalCaseRows: Number(caseStats.total_case_rows || 0),
|
|
distinctCaseNumbers: Number(caseStats.distinct_case_numbers || 0),
|
|
avgRowsPerCase: Number(duplicateRatio.toFixed(2)),
|
|
note: 'ordrestyring_case_latest mangler data'
|
|
}
|
|
});
|
|
}
|
|
|
|
if (Number(hourLinkStats.hours_without_case_case_related || 0) > 0) {
|
|
issues.push({
|
|
issueType: 'HOURS_WITHOUT_CASE',
|
|
severity: 'high',
|
|
entityType: 'hours',
|
|
entityKey: 'global',
|
|
details: {
|
|
hoursWithoutCaseCaseRelated: Number(hourLinkStats.hours_without_case_case_related || 0),
|
|
hoursWithoutCaseAll: Number(hourStats.hours_without_case || 0),
|
|
totalHours: Number(hourStats.total_hours || 0)
|
|
}
|
|
});
|
|
}
|
|
|
|
if (Number(materialStats.materials_without_case || 0) > 0) {
|
|
issues.push({
|
|
issueType: 'MATERIALS_WITHOUT_CASE',
|
|
severity: 'high',
|
|
entityType: 'case_materials',
|
|
entityKey: 'global',
|
|
details: {
|
|
materialsWithoutCase: Number(materialStats.materials_without_case || 0),
|
|
totalMaterials: Number(materialStats.total_materials || 0)
|
|
}
|
|
});
|
|
}
|
|
|
|
const zeroPricePct = Number(materialStats.total_materials || 0) > 0
|
|
? (Number(materialStats.materials_zero_price || 0) / Number(materialStats.total_materials || 1)) * 100
|
|
: 0;
|
|
if (zeroPricePct > 10) {
|
|
issues.push({
|
|
issueType: 'MATERIAL_ZERO_PRICE_RATIO',
|
|
severity: zeroPricePct > 30 ? 'high' : 'medium',
|
|
entityType: 'case_materials',
|
|
entityKey: 'global',
|
|
details: {
|
|
zeroPriceRows: Number(materialStats.materials_zero_price || 0),
|
|
totalRows: Number(materialStats.total_materials || 0),
|
|
percent: Number(zeroPricePct.toFixed(2))
|
|
}
|
|
});
|
|
}
|
|
|
|
for (const issue of issues) {
|
|
await conn.execute(
|
|
`
|
|
INSERT INTO ordrestyring_data_quality_issues (
|
|
issue_type,
|
|
severity,
|
|
entity_type,
|
|
entity_key,
|
|
issue_details_json
|
|
) VALUES (?, ?, ?, ?, ?)
|
|
`,
|
|
[
|
|
issue.issueType,
|
|
issue.severity,
|
|
issue.entityType,
|
|
issue.entityKey,
|
|
JSON.stringify(issue.details)
|
|
]
|
|
);
|
|
}
|
|
|
|
const lines = [];
|
|
lines.push('# Ordrestyring Full Data Quality (Sprint 1)');
|
|
lines.push('');
|
|
lines.push(`Generated: ${new Date().toISOString()}`);
|
|
lines.push('');
|
|
lines.push('## Source tables');
|
|
lines.push(`- cases rows: ${Number(caseStats.total_case_rows || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- distinct cases: ${Number(caseStats.distinct_case_numbers || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- avg rows per case: ${duplicateRatio.toFixed(2)}`);
|
|
lines.push(`- case_latest cache table: ${hasCaseLatestTable ? 'ja' : 'nej'}`);
|
|
lines.push(`- case_latest rows: ${latestCaseRows.toLocaleString('da-DK')}`);
|
|
lines.push(`- hours rows: ${Number(hourStats.total_hours || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- hours without case (all): ${Number(hourStats.hours_without_case || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- hours without case (case-related hour_type): ${Number(hourLinkStats.hours_without_case_case_related || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- hours without remark: ${Number(hourStats.hours_without_remark || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- material rows: ${Number(materialStats.total_materials || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- material rows without text: ${Number(materialStats.materials_without_text || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- material zero-price rows: ${Number(materialStats.materials_zero_price || 0).toLocaleString('da-DK')}`);
|
|
lines.push('');
|
|
lines.push('## Normalized tables');
|
|
lines.push(`- ordrestyring_case_latest rows: ${latestCaseRows.toLocaleString('da-DK')}`);
|
|
lines.push(`- ordrestyring_materials_normalized rows: ${Number(normMatStats.rows_total || 0).toLocaleString('da-DK')}`);
|
|
lines.push(` - m2: ${Number(normMatStats.rows_m2 || 0).toLocaleString('da-DK')}`);
|
|
lines.push(` - lbm: ${Number(normMatStats.rows_lbm || 0).toLocaleString('da-DK')}`);
|
|
lines.push(` - stk: ${Number(normMatStats.rows_stk || 0).toLocaleString('da-DK')}`);
|
|
lines.push(`- ordrestyring_hours_normalized rows: ${Number(normHourStats.rows_total || 0).toLocaleString('da-DK')}`);
|
|
lines.push(` - with duration > 0: ${Number(normHourStats.rows_with_duration || 0).toLocaleString('da-DK')}`);
|
|
lines.push(` - with remark: ${Number(normHourStats.rows_with_remark || 0).toLocaleString('da-DK')}`);
|
|
lines.push('');
|
|
lines.push('## Open quality issues');
|
|
for (const issue of issues) {
|
|
lines.push(`- [${issue.severity}] ${issue.issueType}: ${JSON.stringify(issue.details)}`);
|
|
}
|
|
lines.push('');
|
|
|
|
await fs.mkdir(path.dirname(REPORT_PATH), { recursive: true });
|
|
await fs.writeFile(REPORT_PATH, `${lines.join('\n')}\n`, 'utf8');
|
|
|
|
console.log(JSON.stringify({
|
|
reportPath: REPORT_PATH,
|
|
openIssues: issues.length,
|
|
topIssue: issues[0] || null
|
|
}, null, 2));
|
|
} finally {
|
|
await conn.end();
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|