[verified] test(ordrestyring): add semantic data audit
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env node
|
||||
const path = require('path');
|
||||
const { auditOrdrestyringData, unavailableAudit } = require('../src/services/ordrestyringDataAudit');
|
||||
|
||||
// No sync/bootstrap imports: this command has no mutation mode or schema setup.
|
||||
async function run({ args = [], env = process.env,
|
||||
connect = options => require('mysql2/promise').createConnection(options),
|
||||
write = value => process.stdout.write(value) } = {}) {
|
||||
let connection;
|
||||
let result = unavailableAudit();
|
||||
let timer;
|
||||
try {
|
||||
const port = Number(env.DB_PORT || 3306);
|
||||
const appDatabase = env.DB_NAME || 'tilbudgivern';
|
||||
if (args.length || !env.DB_PASSWORD || !Number.isInteger(port) || port < 1 || port > 65535
|
||||
|| !/^[a-zA-Z0-9_]{1,64}$/.test(appDatabase)) throw new Error('Invalid configuration');
|
||||
connection = await connect({ host: env.DB_HOST || '127.0.0.1', port,
|
||||
user: env.DB_USER || 'tilbudgivern_service', password: env.DB_PASSWORD,
|
||||
database: 'ordrestyring_local', multipleStatements: false, connectTimeout: 5000,
|
||||
supportBigNumbers: true, bigNumberStrings: true });
|
||||
result = await auditOrdrestyringData(connection, { appDatabase });
|
||||
} catch (_) {
|
||||
result = unavailableAudit();
|
||||
} finally {
|
||||
if (connection) {
|
||||
try {
|
||||
await Promise.race([connection.end(), new Promise((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error('Close timeout')), 5000);
|
||||
})]);
|
||||
} catch (_) {
|
||||
try { connection.destroy(); } catch (_) { /* No raw driver output. */ }
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
write(`${JSON.stringify(result)}\n`);
|
||||
return result.status === 'ok' ? 0 : 1;
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
// Matches backend scripts; .env is the existing external-data symlink.
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env'), quiet: true });
|
||||
run({ args: process.argv.slice(2) }).then(code => { process.exitCode = code; });
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
@@ -0,0 +1,310 @@
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { spawn, spawnSync } = require('child_process');
|
||||
const mysql = require('mysql2/promise');
|
||||
const { auditOrdrestyringData } = require('../ordrestyringDataAudit');
|
||||
const { run } = require('../../../scripts/audit-ordrestyring-data-semantics');
|
||||
|
||||
const privateText = 'PRIVATE name [email protected] address remark token';
|
||||
let directory;
|
||||
let server;
|
||||
let connection;
|
||||
let queries;
|
||||
let reader;
|
||||
|
||||
beforeAll(async () => {
|
||||
directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ordrestyring-audit-test-'));
|
||||
const install = spawnSync('mariadb-install-db', [
|
||||
'--no-defaults', `--datadir=${directory}`, '--auth-root-authentication-method=normal',
|
||||
'--skip-test-db'
|
||||
], { encoding: 'utf8' });
|
||||
if (install.status !== 0) throw new Error('Isolated fixture initialization failed');
|
||||
const socketPath = path.join(directory, 'db.sock');
|
||||
server = spawn('mariadbd', ['--no-defaults', `--datadir=${directory}`,
|
||||
`--socket=${socketPath}`, `--pid-file=${directory}/db.pid`, '--skip-networking',
|
||||
`--user=${os.userInfo().username}`, '--innodb-buffer-pool-size=32M'], { stdio: 'ignore' });
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
try {
|
||||
connection = await mysql.createConnection({ socketPath, user: 'root', multipleStatements: true });
|
||||
break;
|
||||
} catch (_) {
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
}
|
||||
if (!connection) throw new Error('Isolated fixture did not start');
|
||||
await connection.query(`
|
||||
CREATE DATABASE ordrestyring_local;
|
||||
CREATE DATABASE audit_app;
|
||||
CREATE TABLE ordrestyring_local.cases (id INT PRIMARY KEY, case_number VARCHAR(255),
|
||||
customer_number VARCHAR(255), main_technician INT, offer_number INT);
|
||||
CREATE TABLE ordrestyring_local.debtors (id INT PRIMARY KEY, customer_number VARCHAR(255));
|
||||
CREATE TABLE ordrestyring_local.users (id INT PRIMARY KEY);
|
||||
CREATE TABLE ordrestyring_local.hours (id INT PRIMARY KEY, emp_id INT,
|
||||
new_case_number VARCHAR(255), exported INT, start_time BIGINT, stop_time BIGINT);
|
||||
CREATE TABLE ordrestyring_local.ordrestyring_hour_mirror_ids (hour_id INT PRIMARY KEY, last_seen_at BIGINT);
|
||||
CREATE TABLE ordrestyring_local.case_materials (id INT PRIMARY KEY, case_number VARCHAR(255),
|
||||
created_by INT, quantity INT, cost_price VARCHAR(255), list_price VARCHAR(255), sales_price VARCHAR(255));
|
||||
CREATE TABLE ordrestyring_local.ordrestyring_import_log (id INT PRIMARY KEY,
|
||||
operation VARCHAR(255), status VARCHAR(255), start_time DATETIME(3), end_time DATETIME(3),
|
||||
created_at DATETIME(3), error_message TEXT);
|
||||
CREATE TABLE audit_app.ordrestyring_offer_snapshots (offer_id INT PRIMARY KEY, offer_number VARCHAR(64));
|
||||
CREATE TABLE audit_app.ordrestyring_offer_line_snapshots (id INT PRIMARY KEY, offer_id INT);
|
||||
CREATE TABLE audit_app.ordrestyring_case_latest (case_number VARCHAR(255) PRIMARY KEY, offer_number INT);
|
||||
INSERT INTO ordrestyring_local.cases VALUES
|
||||
(1,'A','C',1,10),(2,'B','C',1,10),(3,'B','C',1,10),
|
||||
(4,'D','X',9,99),(5,'D','X',9,99),(6,'D','X',9,99),(7,'D','X',9,99),
|
||||
(8,NULL,NULL,NULL,NULL);
|
||||
INSERT INTO ordrestyring_local.debtors VALUES (1,'C'),(2,'E'),(3,'E'),(4,'E'),(5,NULL);
|
||||
INSERT INTO ordrestyring_local.users VALUES (1);
|
||||
INSERT INTO ordrestyring_local.hours VALUES
|
||||
(1,1,'B',0,100,200),(2,9,'missing',1,200,100),(3,NULL,NULL,NULL,100,NULL),
|
||||
(4,1,'A',-1,100,100),(5,1,'A',7,NULL,200);
|
||||
INSERT INTO ordrestyring_local.ordrestyring_hour_mirror_ids VALUES (1,1),(4,1),(99,1);
|
||||
INSERT INTO ordrestyring_local.case_materials VALUES
|
||||
(1,'B',1,2,'12.5','0','-2'),(2,'missing',9,-1,'bad','-1','3'),
|
||||
(3,NULL,NULL,0,NULL,'1','0'),(4,'A',1,NULL,'0',NULL,'bad');
|
||||
INSERT INTO audit_app.ordrestyring_offer_snapshots VALUES (1,'10'),(2,'20');
|
||||
INSERT INTO audit_app.ordrestyring_offer_line_snapshots VALUES (1,1),(2,999);
|
||||
INSERT INTO audit_app.ordrestyring_case_latest VALUES ('A',10),('B',10),('D',99);
|
||||
INSERT INTO ordrestyring_local.ordrestyring_import_log VALUES
|
||||
(1,'sync','success','2026-01-01','2026-01-01 00:00:02.500','2026-01-01',NULL),
|
||||
(2,'sync','no_changes','2026-01-01','2026-01-01 00:00:03.250','2026-01-01',NULL),
|
||||
(3,'other','error','2026-02-01','2026-02-01','2026-02-01',NULL);
|
||||
`);
|
||||
}, 30000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (connection) await connection.end();
|
||||
if (server && server.exitCode === null) {
|
||||
const stopped = new Promise(resolve => server.once('exit', resolve));
|
||||
server.kill('SIGTERM');
|
||||
await stopped;
|
||||
}
|
||||
if (directory) fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
queries = [];
|
||||
reader = {
|
||||
execute: async (options, params) => {
|
||||
queries.push({ ...options, params });
|
||||
return connection.execute(options, params);
|
||||
},
|
||||
destroy: jest.fn()
|
||||
};
|
||||
});
|
||||
|
||||
const audit = () => auditOrdrestyringData(reader, { appDatabase: 'audit_app' });
|
||||
|
||||
test('returns deterministic aggregate counts and exact duplicate percentiles without identifiers', async () => {
|
||||
const result = await audit();
|
||||
expect(result.status).toBe('ok');
|
||||
expect(result.schemaVersion).toBe(1);
|
||||
expect(result.error).toBeNull();
|
||||
expect(result.cases).toEqual({ total: 8, distinct: 3, missingKey: 1,
|
||||
duplicates: { groups: 2, excessRows: 4, min: 1, median: 2, p95: 4, max: 4 } });
|
||||
expect(result.debtors).toEqual({ total: 5, distinct: 2, missingKey: 1,
|
||||
duplicates: { groups: 1, excessRows: 2, min: 1, median: 2, p95: 3, max: 3 } });
|
||||
expect(await audit()).toEqual(result);
|
||||
});
|
||||
|
||||
test('counts all signs, missing and invalid money without coercing malformed values to zero', async () => {
|
||||
expect((await audit()).materials).toEqual({ total: 4,
|
||||
quantity: { negative: 1, zero: 1, positive: 1, missing: 1, invalid: 0 },
|
||||
costPrice: { negative: 0, zero: 1, positive: 1, missing: 1, invalid: 1 },
|
||||
listPrice: { negative: 1, zero: 1, positive: 1, missing: 1, invalid: 0 },
|
||||
salesPrice: { negative: 1, zero: 1, positive: 1, missing: 0, invalid: 1 }
|
||||
});
|
||||
});
|
||||
|
||||
test('separates planning, tracked and unknown provenance and finds tracking inconsistencies', async () => {
|
||||
expect((await audit()).hours).toEqual({ total: 5, localPlanning: 2,
|
||||
trackedNonPlanning: 1, untrackedNonPlanning: 2, trackedPlanning: 1,
|
||||
trackingWithoutHour: 1, trackingTotal: 3, unexpectedExported: 1,
|
||||
missingDuration: 2, negativeDuration: 1, zeroDuration: 1, positiveDuration: 1 });
|
||||
});
|
||||
|
||||
test('counts orphan references once despite duplicate parents, distinguishing missing references', async () => {
|
||||
expect((await audit()).orphans).toEqual({ hoursWithoutUser: 1, hoursWithoutCase: 1,
|
||||
casesWithoutUser: 4, casesWithoutDebtor: 4, casesWithoutOffer: 4,
|
||||
materialsWithoutUser: 1, materialsWithoutCase: 1, offersWithoutCase: 1,
|
||||
offerLinesWithoutOffer: 1 });
|
||||
const sql = queries.map(query => query.sql).join('\n');
|
||||
expect(sql.match(/NOT EXISTS/gi).length).toBeGreaterThanOrEqual(9);
|
||||
expect(sql).not.toMatch(/\bJOIN\b/i);
|
||||
expect(queries.filter(query => /AS orphanCount/i.test(query.sql))).toHaveLength(9);
|
||||
expect(sql).toMatch(/offersWithoutCase[\s\S]*ordrestyring_case_latest|ordrestyring_case_latest[\s\S]*offer_number/i);
|
||||
});
|
||||
|
||||
test('uses only authoritative sync ledger, deterministic ties and millisecond duration', async () => {
|
||||
expect((await audit()).latestLedger).toEqual({ status: 'no_changes', durationMs: 3250 });
|
||||
expect(queries.some(query => query.params.includes('sync'))).toBe(true);
|
||||
await connection.execute('UPDATE ordrestyring_local.ordrestyring_import_log SET status = ?, error_message = ? WHERE id = 2', [privateText, privateText]);
|
||||
try {
|
||||
const result = await audit();
|
||||
expect(result.latestLedger.status).toBe('unknown');
|
||||
expect(JSON.stringify(result)).not.toContain(privateText);
|
||||
} finally {
|
||||
await connection.query("UPDATE ordrestyring_local.ordrestyring_import_log SET status = 'no_changes' WHERE id = 2");
|
||||
}
|
||||
});
|
||||
|
||||
test('executes bounded read-only SQL in a consistent snapshot with no mutation statements', async () => {
|
||||
await audit();
|
||||
expect(queries.length).toBeLessThanOrEqual(20);
|
||||
expect(queries.map(query => query.sql).join('\n')).not.toMatch(/\b(INSERT|UPDATE|DELETE|ALTER|CREATE|TRUNCATE|REPLACE)\b/i);
|
||||
expect(queries.some(query => /WITH CONSISTENT SNAPSHOT, READ ONLY/i.test(query.sql))).toBe(true);
|
||||
for (const query of queries) {
|
||||
expect(query.timeout).toBeGreaterThan(0);
|
||||
expect(query.timeout).toBeLessThanOrEqual(30000);
|
||||
if (/SELECT/i.test(query.sql)) expect(query.sql).toMatch(/^SET STATEMENT max_statement_time = 30 FOR /);
|
||||
}
|
||||
});
|
||||
|
||||
test('fails closed on unsafe schema identifiers without issuing SQL', async () => {
|
||||
const result = await auditOrdrestyringData(reader, { appDatabase: 'bad`; SELECT secret' });
|
||||
expect(result.status).toBe('unavailable');
|
||||
expect(result.error).toBe('AUDIT_UNAVAILABLE');
|
||||
expect(queries).toHaveLength(0);
|
||||
});
|
||||
|
||||
test.each(['ER_NO_SUCH_TABLE', 'ER_BAD_FIELD_ERROR', 'ER_ACCESS_DENIED_ERROR', 'ER_QUERY_TIMEOUT'])(
|
||||
'redacts %s failures and discards partial results', async code => {
|
||||
reader.execute = jest.fn().mockRejectedValue(Object.assign(new Error(privateText), { code }));
|
||||
const result = await audit();
|
||||
expect(result).toEqual({ schemaVersion: 1, status: 'unavailable', error: 'AUDIT_UNAVAILABLE',
|
||||
cases: null, debtors: null, materials: null, hours: null, orphans: null, latestLedger: null });
|
||||
expect(reader.destroy).toHaveBeenCalledTimes(1);
|
||||
}
|
||||
);
|
||||
|
||||
test('command emits one JSON document, closes connections and never prints raw failures', async () => {
|
||||
const output = [];
|
||||
const fakeConnection = { execute: jest.fn().mockRejectedValue(new Error(privateText)), destroy: jest.fn(), end: jest.fn() };
|
||||
const connect = jest.fn().mockResolvedValue(fakeConnection);
|
||||
const code = await run({ env: { DB_PASSWORD: privateText, DB_NAME: 'audit_app' },
|
||||
connect, write: value => output.push(value) });
|
||||
expect(code).toBe(1);
|
||||
expect(output).toHaveLength(1);
|
||||
expect(JSON.parse(output[0]).error).toBe('AUDIT_UNAVAILABLE');
|
||||
expect(output[0]).not.toContain(privateText);
|
||||
expect(connect.mock.calls[0][0]).toMatchObject({ multipleStatements: false, connectTimeout: 5000 });
|
||||
expect(fakeConnection.end).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('command rejects arguments and missing credentials before connecting', async () => {
|
||||
const connect = jest.fn();
|
||||
for (const args of [[], ['--apply']]) {
|
||||
const output = [];
|
||||
expect(await run({ args, env: {}, connect, write: value => output.push(value) })).toBe(1);
|
||||
expect(JSON.parse(output[0]).status).toBe('unavailable');
|
||||
}
|
||||
expect(connect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('database itself rejects writes inside the audit snapshot', async () => {
|
||||
const execute = reader.execute;
|
||||
reader.execute = async (options, params) => {
|
||||
const response = await execute(options, params);
|
||||
if (options.sql.startsWith('START TRANSACTION')) {
|
||||
await expect(connection.query('INSERT INTO ordrestyring_local.users VALUES (99)'))
|
||||
.rejects.toMatchObject({ errno: 1792 });
|
||||
}
|
||||
return response;
|
||||
};
|
||||
expect((await audit()).status).toBe('ok');
|
||||
});
|
||||
|
||||
test('production sources contain no mutation statements or application bootstrap imports', () => {
|
||||
for (const filename of [path.join(__dirname, '../ordrestyringDataAudit.js'),
|
||||
path.join(__dirname, '../../../scripts/audit-ordrestyring-data-semantics.js')]) {
|
||||
const source = fs.readFileSync(filename, 'utf8');
|
||||
expect(source).not.toMatch(/\b(INSERT|UPDATE|DELETE|ALTER|CREATE|TRUNCATE|REPLACE)\b/i);
|
||||
expect(source).not.toMatch(/require\([^)]*(?:databaseService|ordrestyringSyncService|logger)/);
|
||||
}
|
||||
});
|
||||
|
||||
test('terminates a stalled query at the client deadline without leaking errors', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
reader.execute = jest.fn(() => new Promise(() => {}));
|
||||
const pending = audit();
|
||||
await jest.advanceTimersByTimeAsync(30000);
|
||||
expect((await pending).status).toBe('unavailable');
|
||||
expect(reader.destroy).toHaveBeenCalledTimes(1);
|
||||
expect(reader.execute).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test.each([privateText, '9007199254740993', -1, Infinity, undefined, ''])(
|
||||
'rejects malformed or unsafe aggregate values (%s)', async value => {
|
||||
const execute = reader.execute;
|
||||
reader.execute = async (options, params) => {
|
||||
const response = await execute(options, params);
|
||||
if (options.sql.includes('WITH counts AS')) response[0][0].total = value;
|
||||
return response;
|
||||
};
|
||||
const result = await audit();
|
||||
expect(result.status).toBe('unavailable');
|
||||
expect(JSON.stringify(result)).not.toContain(privateText);
|
||||
await connection.query('ROLLBACK');
|
||||
}
|
||||
);
|
||||
|
||||
test('command success is the same aggregate document and closes its dedicated connection', async () => {
|
||||
const output = [];
|
||||
reader.end = jest.fn();
|
||||
const expected = await audit();
|
||||
expect(await run({ env: { DB_PASSWORD: privateText, DB_NAME: 'audit_app' },
|
||||
connect: async () => reader, write: value => output.push(value) })).toBe(0);
|
||||
expect(output).toEqual([`${JSON.stringify(expected)}\n`]);
|
||||
expect(reader.end).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('connection and cleanup failures cannot expose credentials or raw errors', async () => {
|
||||
const output = [];
|
||||
expect(await run({ env: { DB_PASSWORD: privateText },
|
||||
connect: async () => { throw new Error(privateText); },
|
||||
write: value => output.push(value) })).toBe(1);
|
||||
expect(output[0]).not.toContain(privateText);
|
||||
reader.end = jest.fn().mockRejectedValue(new Error(privateText));
|
||||
reader.destroy = jest.fn(() => { throw new Error(privateText); });
|
||||
expect(await run({ env: { DB_PASSWORD: privateText, DB_NAME: 'audit_app' },
|
||||
connect: async () => reader, write: value => output.push(value) })).toBe(0);
|
||||
expect(output.join('')).not.toContain(privateText);
|
||||
expect(reader.destroy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('empty datasets produce zeros, null percentiles and unavailable ledger without changing schema', async () => {
|
||||
const tables = ['ordrestyring_local.cases', 'ordrestyring_local.debtors', 'ordrestyring_local.users',
|
||||
'ordrestyring_local.hours', 'ordrestyring_local.ordrestyring_hour_mirror_ids',
|
||||
'ordrestyring_local.case_materials', 'ordrestyring_local.ordrestyring_import_log',
|
||||
'audit_app.ordrestyring_offer_snapshots', 'audit_app.ordrestyring_offer_line_snapshots'];
|
||||
const saved = [];
|
||||
const before = await audit();
|
||||
try {
|
||||
for (const table of tables) {
|
||||
const [rows] = await connection.query(`SELECT * FROM ${table}`);
|
||||
saved.push([table, rows]);
|
||||
await connection.query(`DELETE FROM ${table}`);
|
||||
}
|
||||
const result = await audit();
|
||||
expect(Object.keys(result)).toEqual(Object.keys(before));
|
||||
expect(result.status).toBe('ok');
|
||||
expect(result.cases).toEqual({ total: 0, distinct: 0, missingKey: 0,
|
||||
duplicates: { groups: 0, excessRows: 0, min: null, median: null, p95: null, max: null } });
|
||||
expect(result.debtors).toEqual(result.cases);
|
||||
expect(Object.values(result.hours)).toEqual(Object.values(result.hours).map(() => 0));
|
||||
expect(Object.values(result.orphans)).toEqual(Object.values(result.orphans).map(() => 0));
|
||||
expect(result.materials.total).toBe(0);
|
||||
expect(result.materials.salesPrice).toEqual({ negative: 0, zero: 0, positive: 0, missing: 0, invalid: 0 });
|
||||
expect(result.latestLedger).toEqual({ status: 'unavailable', durationMs: null });
|
||||
} finally {
|
||||
for (const [table, rows] of saved) {
|
||||
for (const row of rows) await connection.query(`INSERT INTO ${table} SET ?`, row);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
const QUERY_TIMEOUT_MS = 30000;
|
||||
|
||||
function unavailableAudit() {
|
||||
return { schemaVersion: 1, status: 'unavailable', error: 'AUDIT_UNAVAILABLE',
|
||||
cases: null, debtors: null, materials: null, hours: null, orphans: null, latestLedger: null };
|
||||
}
|
||||
|
||||
// Only aggregate numbers and explicitly allowed status values may leave this module.
|
||||
function numeric(value, nullable = false) {
|
||||
if (value === null && nullable) return null;
|
||||
if ((typeof value !== 'number' && typeof value !== 'string') || value === '') {
|
||||
throw new Error('Invalid aggregate');
|
||||
}
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number < 0 || number > Number.MAX_SAFE_INTEGER) {
|
||||
throw new Error('Invalid aggregate');
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function numbers(row, fields) {
|
||||
return Object.fromEntries(fields.map(field => [field, numeric(row[field])]));
|
||||
}
|
||||
|
||||
function duplicateSql(table, key) {
|
||||
// Distribution covers rows per non-null business key (including singleton keys).
|
||||
// Median averages the two central observations; p95 uses the nearest rank.
|
||||
return `WITH counts AS (
|
||||
SELECT COUNT(*) AS n FROM ordrestyring_local.${table}
|
||||
WHERE ${key} IS NOT NULL GROUP BY ${key}
|
||||
), ranked AS (
|
||||
SELECT n, ROW_NUMBER() OVER (ORDER BY n) AS rn, COUNT(*) OVER () AS population FROM counts
|
||||
) SELECT
|
||||
(SELECT COUNT(*) FROM ordrestyring_local.${table}) AS total,
|
||||
COUNT(*) AS distinctCount,
|
||||
(SELECT COUNT(*) FROM ordrestyring_local.${table} WHERE ${key} IS NULL) AS missingKey,
|
||||
COALESCE(SUM(n > 1), 0) AS duplicateGroups, COALESCE(SUM(n - 1), 0) AS excessRows,
|
||||
MIN(n) AS minimum, AVG(CASE WHEN rn IN (FLOOR((population + 1) / 2),
|
||||
FLOOR((population + 2) / 2)) THEN n END) AS median,
|
||||
MAX(CASE WHEN rn = CEIL(population * 0.95) THEN n END) AS p95, MAX(n) AS maximum
|
||||
FROM ranked`;
|
||||
}
|
||||
|
||||
function duplicateResult(row) {
|
||||
return { total: numeric(row.total), distinct: numeric(row.distinctCount),
|
||||
missingKey: numeric(row.missingKey), duplicates: {
|
||||
groups: numeric(row.duplicateGroups), excessRows: numeric(row.excessRows),
|
||||
min: numeric(row.minimum, true), median: numeric(row.median, true),
|
||||
p95: numeric(row.p95, true), max: numeric(row.maximum, true)
|
||||
} };
|
||||
}
|
||||
|
||||
const MATERIAL_FIELDS = { quantity: 'quantity', costPrice: 'cost_price',
|
||||
listPrice: 'list_price', salesPrice: 'sales_price' };
|
||||
const SIGNS = ['negative', 'zero', 'positive', 'missing', 'invalid'];
|
||||
|
||||
function materialSql() {
|
||||
const columns = Object.entries(MATERIAL_FIELDS).flatMap(([name, column]) => {
|
||||
const valid = `TRIM(${column}) REGEXP '^[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)$'`;
|
||||
// Guard the cast: malformed strings must never be counted as zero prices.
|
||||
const value = `CASE WHEN ${valid} THEN CAST(${column} AS DECIMAL(40,10)) END`;
|
||||
const predicates = [`${value} < 0`, `${value} = 0`, `${value} > 0`,
|
||||
`${column} IS NULL`, `${column} IS NOT NULL AND NOT (${valid})`];
|
||||
return predicates.map((predicate, index) =>
|
||||
`COALESCE(SUM(${predicate}), 0) AS ${name}_${SIGNS[index]}`);
|
||||
});
|
||||
return `SELECT COUNT(*) AS total, ${columns.join(', ')} FROM ordrestyring_local.case_materials`;
|
||||
}
|
||||
|
||||
const HOURS_FIELDS = ['total', 'localPlanning', 'trackedNonPlanning', 'untrackedNonPlanning',
|
||||
'trackedPlanning', 'trackingWithoutHour', 'trackingTotal', 'unexpectedExported',
|
||||
'missingDuration', 'negativeDuration', 'zeroDuration', 'positiveDuration'];
|
||||
|
||||
function hoursSql() {
|
||||
const planning = '(h.exported IS NULL OR h.exported = -1)';
|
||||
const tracked = 'EXISTS (SELECT 1 FROM ordrestyring_local.ordrestyring_hour_mirror_ids t WHERE t.hour_id = h.id)';
|
||||
return `SELECT COUNT(*) AS total,
|
||||
COALESCE(SUM(${planning}), 0) AS localPlanning,
|
||||
COALESCE(SUM(NOT ${planning} AND ${tracked}), 0) AS trackedNonPlanning,
|
||||
COALESCE(SUM(NOT ${planning} AND NOT ${tracked}), 0) AS untrackedNonPlanning,
|
||||
COALESCE(SUM(${planning} AND ${tracked}), 0) AS trackedPlanning,
|
||||
(SELECT COUNT(*) FROM ordrestyring_local.ordrestyring_hour_mirror_ids t
|
||||
WHERE NOT EXISTS (SELECT 1 FROM ordrestyring_local.hours h2 WHERE h2.id = t.hour_id)) AS trackingWithoutHour,
|
||||
(SELECT COUNT(*) FROM ordrestyring_local.ordrestyring_hour_mirror_ids) AS trackingTotal,
|
||||
COALESCE(SUM(h.exported NOT IN (-1, 0, 1)), 0) AS unexpectedExported,
|
||||
COALESCE(SUM(h.start_time IS NULL OR h.stop_time IS NULL), 0) AS missingDuration,
|
||||
COALESCE(SUM(h.stop_time < h.start_time), 0) AS negativeDuration,
|
||||
COALESCE(SUM(h.stop_time = h.start_time), 0) AS zeroDuration,
|
||||
COALESCE(SUM(h.stop_time > h.start_time), 0) AS positiveDuration
|
||||
FROM ordrestyring_local.hours h`;
|
||||
}
|
||||
|
||||
function orphanRelations(app) {
|
||||
// Null references are absent, not dangling. Counts refer to child rows, not distinct keys.
|
||||
// An offer without a case is unmatched, and need not indicate corrupt data.
|
||||
const local = 'ordrestyring_local';
|
||||
return [
|
||||
['hoursWithoutUser', `${local}.hours`, 'emp_id', `${local}.users`, 'id'],
|
||||
['hoursWithoutCase', `${local}.hours`, 'new_case_number', `${local}.cases`, 'case_number'],
|
||||
['casesWithoutUser', `${local}.cases`, 'main_technician', `${local}.users`, 'id'],
|
||||
['casesWithoutDebtor', `${local}.cases`, 'customer_number', `${local}.debtors`, 'customer_number'],
|
||||
['casesWithoutOffer', `${local}.cases`, 'offer_number', `${app}.ordrestyring_offer_snapshots`, 'offer_number'],
|
||||
['materialsWithoutUser', `${local}.case_materials`, 'created_by', `${local}.users`, 'id'],
|
||||
['materialsWithoutCase', `${local}.case_materials`, 'case_number', `${local}.cases`, 'case_number'],
|
||||
['offersWithoutCase', `${app}.ordrestyring_offer_snapshots`, 'offer_number', `${app}.ordrestyring_case_latest`, 'offer_number'],
|
||||
['offerLinesWithoutOffer', `${app}.ordrestyring_offer_line_snapshots`, 'offer_id', `${app}.ordrestyring_offer_snapshots`, 'offer_id']
|
||||
];
|
||||
}
|
||||
|
||||
// Requires a dedicated connection: failures destroy it, releasing the read-only snapshot.
|
||||
// Nine sequential statements, each capped at five seconds; no pagination or row exports.
|
||||
async function auditOrdrestyringData(connection, { appDatabase = 'tilbudgivern' } = {}) {
|
||||
const result = unavailableAudit();
|
||||
if (typeof appDatabase !== 'string' || !/^[a-zA-Z0-9_]{1,64}$/.test(appDatabase)) return result;
|
||||
const query = async (sql, params = [], aggregate = true) => {
|
||||
let timer;
|
||||
try {
|
||||
const boundedSql = aggregate ? `SET STATEMENT max_statement_time = 30 FOR ${sql}` : sql;
|
||||
return await Promise.race([
|
||||
connection.execute({ sql: boundedSql, timeout: QUERY_TIMEOUT_MS }, params),
|
||||
new Promise((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error('Audit timeout')), QUERY_TIMEOUT_MS);
|
||||
})
|
||||
]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
const row = async (sql, params) => {
|
||||
const [rows] = await query(sql, params);
|
||||
if (rows.length > 1) throw new Error('Unbounded aggregate');
|
||||
return rows[0];
|
||||
};
|
||||
try {
|
||||
await query('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ', [], false);
|
||||
await query('START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY', [], false);
|
||||
result.cases = duplicateResult(await row(duplicateSql('cases', 'case_number')));
|
||||
result.debtors = duplicateResult(await row(duplicateSql('debtors', 'customer_number')));
|
||||
const materials = await row(materialSql());
|
||||
result.materials = { total: numeric(materials.total) };
|
||||
for (const field of Object.keys(MATERIAL_FIELDS)) {
|
||||
result.materials[field] = Object.fromEntries(SIGNS.map(sign => [sign, numeric(materials[`${field}_${sign}`])]));
|
||||
}
|
||||
result.hours = numbers(await row(hoursSql()), HOURS_FIELDS);
|
||||
const relations = orphanRelations(`\`${appDatabase}\``);
|
||||
const orphanCounts = {};
|
||||
for (const [name, child, reference, parent, key] of relations) {
|
||||
const orphanRow = await row(`SELECT COUNT(*) AS orphanCount
|
||||
FROM ${child} child
|
||||
WHERE child.${reference} IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM ${parent} parent WHERE parent.${key} = child.${reference})`);
|
||||
orphanCounts[name] = numeric(orphanRow.orphanCount);
|
||||
}
|
||||
result.orphans = orphanCounts;
|
||||
const ledger = await row(`SELECT
|
||||
CASE WHEN status IN ('success', 'no_changes', 'error') THEN status ELSE 'unknown' END AS status,
|
||||
CASE WHEN end_time >= start_time THEN TIMESTAMPDIFF(MICROSECOND, start_time, end_time) / 1000
|
||||
ELSE NULL END AS durationMs
|
||||
FROM ordrestyring_local.ordrestyring_import_log WHERE operation = ?
|
||||
ORDER BY created_at DESC, id DESC LIMIT 1`, ['sync']);
|
||||
result.latestLedger = ledger ? {
|
||||
status: ['success', 'no_changes', 'error'].includes(ledger.status) ? ledger.status : 'unknown',
|
||||
durationMs: numeric(ledger.durationMs, true)
|
||||
} : { status: 'unavailable', durationMs: null };
|
||||
await query('COMMIT', [], false);
|
||||
result.status = 'ok';
|
||||
result.error = null;
|
||||
return result;
|
||||
} catch (_) {
|
||||
try { connection.destroy(); } catch (_) { /* Never expose driver errors. */ }
|
||||
return unavailableAudit();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { auditOrdrestyringData, unavailableAudit };
|
||||
Reference in New Issue
Block a user