alex
c3731a470f
feat: Add standalone script for enhanced Bygma CSV import with validation test: Create simple database test to show current state and product counts test: Develop enhanced validation demo to simulate problematic CSV data handling test: Implement price validation tests with higher limits for edge cases test: Conduct real enhanced validation test with actual Bygma CSV file test: Test validation improvements with known problematic data test: Validate logic of cleaning and validating CSV rows without database test: Test improved VareNr cleaning functionality to remove spaces
260 lines
9.4 KiB
JavaScript
260 lines
9.4 KiB
JavaScript
/**
|
|
* Direct CSV analysis to show what our enhanced validation would catch
|
|
* Without database dependency
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const csv = require('csv-parser');
|
|
const { Transform } = require('stream');
|
|
|
|
// Our enhanced validation functions (copied from the service)
|
|
function cleanString(str) {
|
|
if (!str) return null;
|
|
return str.toString()
|
|
.replace(/^\uFEFF/, '') // Remove BOM
|
|
.replace(/\r?\n/g, ' ') // Replace line breaks with space
|
|
.replace(/\r/g, '') // Remove remaining carriage returns
|
|
.replace(/\t/g, ' ') // Replace tabs with space
|
|
.replace(/\s+/g, ' ') // Replace multiple spaces with single space
|
|
.replace(/[^\x20-\x7E\u00A0-\uFFFF]/g, '') // Remove control characters but keep unicode
|
|
.trim(); // Trim whitespace
|
|
}
|
|
|
|
function parsePrice(priceStr) {
|
|
if (!priceStr || priceStr === '' || priceStr === null || priceStr === undefined) return 0;
|
|
|
|
let cleaned = priceStr.toString().trim();
|
|
|
|
if (cleaned === '0' || cleaned === '0,00' || cleaned === '0.00') return 0;
|
|
if (cleaned === '' || cleaned === '-' || cleaned === 'N/A' || cleaned === 'NULL') return 0;
|
|
|
|
cleaned = cleaned
|
|
.replace(/DKK|Kr\.?|kr\.?/gi, '')
|
|
.replace(/[^\d,.-]/g, '')
|
|
.replace(/^-+|--+|-+$/g, '')
|
|
.trim();
|
|
|
|
if (!cleaned || cleaned === '-') return 0;
|
|
|
|
if (cleaned.includes(',') && cleaned.includes('.')) {
|
|
const lastComma = cleaned.lastIndexOf(',');
|
|
const lastDot = cleaned.lastIndexOf('.');
|
|
if (lastDot > lastComma) {
|
|
cleaned = cleaned.replace(/,/g, '');
|
|
} else {
|
|
cleaned = cleaned.replace(/\./g, '').replace(',', '.');
|
|
}
|
|
} else if (cleaned.includes(',')) {
|
|
cleaned = cleaned.replace(',', '.');
|
|
}
|
|
|
|
const parsed = parseFloat(cleaned);
|
|
return isNaN(parsed) || parsed < 0 ? 0 : Math.round(parsed * 100) / 100;
|
|
}
|
|
|
|
function enhancedValidateRow(row, lineNumber) {
|
|
// Clean all fields first
|
|
const cleanedRow = {};
|
|
for (const [key, value] of Object.entries(row)) {
|
|
cleanedRow[key] = cleanString(value);
|
|
}
|
|
|
|
// Skip completely empty rows
|
|
const hasAnyData = Object.values(cleanedRow).some(value => value && value.trim() !== '');
|
|
if (!hasAnyData) {
|
|
return { valid: false, error: 'Empty row - skipping', severity: 'filtered' };
|
|
}
|
|
|
|
// Advanced validation for malformed CSV rows
|
|
if (cleanedRow.VareNr && (cleanedRow.VareNr.includes('STK;') || cleanedRow.VareNr.includes(';'))) {
|
|
return { valid: false, error: 'Malformed CSV row with merged columns', severity: 'filtered' };
|
|
}
|
|
|
|
if (cleanedRow.VareNr && cleanedRow.VareNr.length > 50) {
|
|
return { valid: false, error: 'VareNr too long - likely malformed row', severity: 'filtered' };
|
|
}
|
|
|
|
if (cleanedRow.Tekst && cleanedRow.Tekst.length > 500) {
|
|
return { valid: false, error: 'Tekst field too long - likely data corruption', severity: 'filtered' };
|
|
}
|
|
|
|
// Strict validation of required fields
|
|
if (!cleanedRow.VareNr || !cleanedRow.Tekst || !cleanedRow.Enhed ||
|
|
cleanedRow.VareNr.trim() === '' || cleanedRow.Tekst.trim() === '' || cleanedRow.Enhed.trim() === '') {
|
|
|
|
const missing = [];
|
|
if (!cleanedRow.VareNr || cleanedRow.VareNr.trim() === '') missing.push('VareNr');
|
|
if (!cleanedRow.Tekst || cleanedRow.Tekst.trim() === '') missing.push('Tekst');
|
|
if (!cleanedRow.Enhed || cleanedRow.Enhed.trim() === '') missing.push('Enhed');
|
|
|
|
return { valid: false, error: `Missing required fields: ${missing.join(', ')}`, severity: 'error' };
|
|
}
|
|
|
|
if (cleanedRow.Tekst.includes(';;') || cleanedRow.Tekst.includes('\t')) {
|
|
return { valid: false, error: 'Invalid field format - contains unexpected characters', severity: 'error' };
|
|
}
|
|
|
|
if (cleanedRow.VareNr.length > 20 || cleanedRow.VareNr.length < 3) {
|
|
return { valid: false, error: `Invalid VareNr format - length ${cleanedRow.VareNr.length} (expected 3-20 chars)`, severity: 'error' };
|
|
}
|
|
|
|
if (!/^[A-Za-z0-9\-_\/\.\s]+$/.test(cleanedRow.VareNr)) {
|
|
return { valid: false, error: 'Invalid VareNr format - contains invalid characters', severity: 'error' };
|
|
}
|
|
|
|
if (cleanedRow.Enhed.length > 10) {
|
|
return { valid: false, error: 'Invalid Enhed - too long', severity: 'error' };
|
|
}
|
|
|
|
// Test price parsing
|
|
const bruttoPris = parsePrice(cleanedRow.BruttoPris);
|
|
const nettoPris = parsePrice(cleanedRow.Nettopris);
|
|
|
|
if (bruttoPris <= 0 && nettoPris <= 0) {
|
|
return { valid: false, error: 'Both prices are missing or zero', severity: 'error' };
|
|
}
|
|
|
|
return { valid: true, cleanedRow };
|
|
}
|
|
|
|
async function analyzeRealCSV() {
|
|
console.log('🔍 Analyzing Real Bygma CSV with Enhanced Validation...\n');
|
|
|
|
const filePath = './frontend/firmadata/subvendor/bygma/PrisBog_20250918090450_38178059_ Tømrer- og Snedker Mikael Holck ApS_ 156.csv';
|
|
|
|
return new Promise((resolve) => {
|
|
let lineNumber = 0;
|
|
let validRows = 0;
|
|
let filteredRows = 0; // Caught by pre-processing
|
|
let errorRows = 0; // Traditional validation errors
|
|
const errors = [];
|
|
const filtered = [];
|
|
|
|
const startTime = Date.now();
|
|
|
|
fs.createReadStream(filePath)
|
|
.pipe(csv({
|
|
separator: ';',
|
|
headers: ['Varegrp', 'VareNr', 'Tekst', 'Enhed', 'BruttoPris', 'Opdat', 'Nettopris', 'DB-nr', 'EAN-nr', 'Std'],
|
|
skipLinesWithError: false,
|
|
strict: false
|
|
}))
|
|
.on('data', (row) => {
|
|
lineNumber++;
|
|
|
|
// Skip header
|
|
if (lineNumber === 1) return;
|
|
|
|
// Enhanced pre-processing validation (what our Transform stream would catch)
|
|
if (!row || typeof row !== 'object') {
|
|
filteredRows++;
|
|
filtered.push({ line: lineNumber, reason: 'Invalid object' });
|
|
return;
|
|
}
|
|
|
|
// Pre-clean chunk data
|
|
const cleanedChunk = {};
|
|
Object.keys(row).forEach(key => {
|
|
if (row[key] && typeof row[key] === 'string') {
|
|
cleanedChunk[key] = row[key].replace(/[\r\n\t\u0000-\u001F\u007F-\u009F]/g, '').trim();
|
|
} else {
|
|
cleanedChunk[key] = row[key];
|
|
}
|
|
});
|
|
|
|
// Advanced malformed row detection
|
|
const nonEmptyFields = Object.values(cleanedChunk).filter(value =>
|
|
value && value.toString().trim() !== ''
|
|
).length;
|
|
|
|
if (nonEmptyFields < 3) {
|
|
filteredRows++;
|
|
filtered.push({ line: lineNumber, reason: 'Too few fields', fields: nonEmptyFields });
|
|
return;
|
|
}
|
|
|
|
// Check for obvious parsing errors
|
|
const hasParsingErrors = Object.values(cleanedChunk).some(value => {
|
|
if (!value || typeof value !== 'string') return false;
|
|
return value.includes(';;') || value.startsWith(';') || value.endsWith(';');
|
|
});
|
|
|
|
if (hasParsingErrors) {
|
|
filteredRows++;
|
|
filtered.push({ line: lineNumber, reason: 'Parsing errors detected' });
|
|
return;
|
|
}
|
|
|
|
// Check for unreasonably long fields
|
|
const hasLongFields = Object.values(cleanedChunk).some(value => {
|
|
if (!value || typeof value !== 'string') return false;
|
|
return value.length > 200;
|
|
});
|
|
|
|
if (hasLongFields) {
|
|
filteredRows++;
|
|
filtered.push({ line: lineNumber, reason: 'Unreasonably long fields' });
|
|
return;
|
|
}
|
|
|
|
// Now do traditional validation
|
|
const validation = enhancedValidateRow(cleanedChunk, lineNumber);
|
|
|
|
if (validation.valid) {
|
|
validRows++;
|
|
} else if (validation.severity === 'filtered') {
|
|
filteredRows++;
|
|
filtered.push({ line: lineNumber, reason: validation.error });
|
|
} else {
|
|
errorRows++;
|
|
errors.push({ line: lineNumber, error: validation.error });
|
|
}
|
|
|
|
// Progress indicator
|
|
if (lineNumber % 10000 === 0) {
|
|
process.stdout.write(`\r📊 Processed ${lineNumber} rows...`);
|
|
}
|
|
})
|
|
.on('end', () => {
|
|
const duration = ((Date.now() - startTime) / 1000).toFixed(1);
|
|
|
|
console.log(`\n\n🎉 Enhanced Validation Analysis Complete!\n`);
|
|
|
|
console.log('📊 Results:');
|
|
console.log(` - Total rows processed: ${lineNumber - 1}`);
|
|
console.log(` - Valid rows: ${validRows}`);
|
|
console.log(` - Pre-filtered rows: ${filteredRows}`);
|
|
console.log(` - Traditional errors: ${errorRows}`);
|
|
console.log(` - Total failed: ${filteredRows + errorRows}`);
|
|
console.log(` - Success rate: ${((validRows / (lineNumber - 1)) * 100).toFixed(1)}%`);
|
|
console.log(` - Duration: ${duration} seconds\n`);
|
|
|
|
// Compare with original
|
|
console.log('📈 Comparison with Original Import:');
|
|
console.log(` - Original failed: 582 rows`);
|
|
console.log(` - Enhanced failed: ${filteredRows + errorRows} rows`);
|
|
console.log(` - Improvement: ${582 - (filteredRows + errorRows)} fewer failures`);
|
|
console.log(` - Reduction: ${(((582 - (filteredRows + errorRows)) / 582) * 100).toFixed(1)}%\n`);
|
|
|
|
if (filtered.length > 0) {
|
|
console.log('🔧 Pre-processing Filtered (first 10):');
|
|
filtered.slice(0, 10).forEach((item, index) => {
|
|
console.log(` ${index + 1}. Line ${item.line}: ${item.reason}`);
|
|
});
|
|
console.log('');
|
|
}
|
|
|
|
if (errors.length > 0) {
|
|
console.log('⚠️ Remaining Errors (first 10):');
|
|
errors.slice(0, 10).forEach((error, index) => {
|
|
console.log(` ${index + 1}. Line ${error.line}: ${error.error}`);
|
|
});
|
|
}
|
|
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
analyzeRealCSV();
|