tilbudgivern/test_validation_logic.js

// Test validation functions directly without database
const fs = require('fs');
const csv = require('csv-parser');
const { Transform } = require('stream');

// Simplified versions of our validation functions for testing
function cleanString(str) {
  if (!str) return null;
  return str.toString()
    .replace(/^\uFEFF/, '') // Remove BOM
    .replace(/\r?\n/g, ' ') // Replace line breaks with space
    .replace(/\r/g, '') // Remove remaining carriage returns
    .replace(/\t/g, ' ') // Replace tabs with space
    .replace(/\s+/g, ' ') // Replace multiple spaces with single space
    .replace(/[^\x20-\x7E\u00A0-\uFFFF]/g, '') // Remove control characters but keep unicode
    .trim(); // Trim whitespace
}

function parsePrice(priceStr) {
  if (!priceStr || priceStr === '' || priceStr === null || priceStr === undefined) return 0;
  
  let cleaned = priceStr.toString().trim();
  
  if (cleaned === '0' || cleaned === '0,00' || cleaned === '0.00') return 0;
  if (cleaned === '' || cleaned === '-' || cleaned === 'N/A' || cleaned === 'NULL') return 0;
  
  cleaned = cleaned
    .replace(/DKK|Kr\.?|kr\.?/gi, '')
    .replace(/[^\d,.-]/g, '')
    .replace(/^-+|--+|-+$/g, '')
    .trim();
  
  if (!cleaned || cleaned === '-') return 0;
  
  if (cleaned.includes(',') && cleaned.includes('.')) {
    const lastComma = cleaned.lastIndexOf(',');
    const lastDot = cleaned.lastIndexOf('.');
    if (lastDot > lastComma) {
      cleaned = cleaned.replace(/,/g, '');
    } else {
      cleaned = cleaned.replace(/\./g, '').replace(',', '.');
    }
  } else if (cleaned.includes(',')) {
    cleaned = cleaned.replace(',', '.');
  }
  
  const parsed = parseFloat(cleaned);
  return isNaN(parsed) || parsed < 0 ? 0 : Math.round(parsed * 100) / 100;
}

function validateRow(row, lineNumber) {
  // Clean all fields first
  const cleanedRow = {};
  for (const [key, value] of Object.entries(row)) {
    cleanedRow[key] = cleanString(value);
  }

  // Skip completely empty rows
  const hasAnyData = Object.values(cleanedRow).some(value => value && value.trim() !== '');
  if (!hasAnyData) {
    return { valid: false, error: 'Empty row - skipping' };
  }

  // Advanced validation for malformed CSV rows
  if (cleanedRow.VareNr && (cleanedRow.VareNr.includes('STK;') || cleanedRow.VareNr.includes(';'))) {
    return { valid: false, error: 'Malformed CSV row with merged columns' };
  }

  if (cleanedRow.VareNr && cleanedRow.VareNr.length > 50) {
    return { valid: false, error: 'VareNr too long - likely malformed row' };
  }

  if (cleanedRow.Tekst && cleanedRow.Tekst.length > 500) {
    return { valid: false, error: 'Tekst field too long - likely data corruption' };
  }

  // Strict validation of required fields
  if (!cleanedRow.VareNr || !cleanedRow.Tekst || !cleanedRow.Enhed || 
      cleanedRow.VareNr.trim() === '' || cleanedRow.Tekst.trim() === '' || cleanedRow.Enhed.trim() === '') {
    
    const missing = [];
    if (!cleanedRow.VareNr || cleanedRow.VareNr.trim() === '') missing.push('VareNr');
    if (!cleanedRow.Tekst || cleanedRow.Tekst.trim() === '') missing.push('Tekst');
    if (!cleanedRow.Enhed || cleanedRow.Enhed.trim() === '') missing.push('Enhed');
    
    return { valid: false, error: `Missing required fields: ${missing.join(', ')}` };
  }

  if (cleanedRow.Tekst.includes(';;') || cleanedRow.Tekst.includes('\t')) {
    return { valid: false, error: 'Invalid field format - contains unexpected characters' };
  }

  if (cleanedRow.VareNr.length > 20 || cleanedRow.VareNr.length < 3) {
    return { valid: false, error: `Invalid VareNr format - length ${cleanedRow.VareNr.length} (expected 3-20 chars)` };
  }

  if (!/^[A-Za-z0-9\-_\/\.]+$/.test(cleanedRow.VareNr)) {
    return { valid: false, error: 'Invalid VareNr format - contains invalid characters' };
  }

  if (cleanedRow.Enhed.length > 10) {
    return { valid: false, error: 'Invalid Enhed - too long' };
  }

  // Test price parsing
  const bruttoPris = parsePrice(cleanedRow.BruttoPris);
  const nettoPris = parsePrice(cleanedRow.Nettopris);
  
  if (bruttoPris <= 0 && nettoPris <= 0) {
    return { valid: false, error: 'Both prices are missing or zero' };
  }

  return { valid: true, cleanedRow };
}

async function testValidationLogic() {
  console.log('🧪 Testing validation logic with problematic data...');
  
  // Create test CSV with known problematic rows
  const testCSV = `Varegrp;VareNr;Tekst;Enhed;BruttoPris;Opdat;Nettopris;DB-nr;EAN-nr;Std
9990;135865;JUNKBAG SÆKKE L M3/1300KG;STK;175,96;1;175,96;;;
;;;;;;;;
9990;;MISSING VARENR;STK;100,00;1;80,00;;;
9990;123456;;STK;50,00;1;40,00;;;
9990;789012;MISSING ENHED;;25,00;1;20,00;;;
9990;TOOLONGVARENUMBERHERE123456789;NORMAL PRODUCT;STK;30,00;1;25,00;;;
9990;999999;PRODUCT WITH SEMICOLONS;;STK;15,00;1;12,00;;;
9990;111111;NORMAL PRODUCT;STK;abc,def;1;invalid;;;
9990;222222;ANOTHER GOOD PRODUCT;STK;200,50;1;160,40;;;
9990;333333;PRODUCT WITH VERY LONG TEXT THAT GOES ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON AND ON;STK;100,00;1;80,00;;;
;;;	;;;;;;
9990;444444;GOOD PRODUCT AFTER PROBLEMS;STK;75,25;1;60,20;;;`;

  // Write test file
  fs.writeFileSync('./test_validation_logic.csv', testCSV);
  
  return new Promise((resolve) => {
    let lineNumber = 0;
    let validRows = 0;
    let invalidRows = 0;
    const errors = [];
    
    fs.createReadStream('./test_validation_logic.csv')
      .pipe(csv({
        separator: ';',
        headers: ['Varegrp', 'VareNr', 'Tekst', 'Enhed', 'BruttoPris', 'Opdat', 'Nettopris', 'DB-nr', 'EAN-nr', 'Std'],
        skipLinesWithError: false,
        strict: false
      }))
      .on('data', (row) => {
        lineNumber++;
        
        // Skip header
        if (lineNumber === 1) return;
        
        const validation = validateRow(row, lineNumber);
        
        if (validation.valid) {
          validRows++;
          console.log(`✅ Line ${lineNumber}: Valid`);
        } else {
          invalidRows++;
          errors.push({ line: lineNumber, error: validation.error });
          console.log(`❌ Line ${lineNumber}: ${validation.error}`);
        }
      })
      .on('end', () => {
        console.log('\n📊 Validation Test Results:');
        console.log(`   - Total rows tested: ${lineNumber - 1}`);
        console.log(`   - Valid rows: ${validRows}`);
        console.log(`   - Invalid rows: ${invalidRows}`);
        console.log(`   - Error rate: ${((invalidRows / (lineNumber - 1)) * 100).toFixed(1)}%`);
        
        console.log('\n🎯 Our enhanced validation successfully:');
        console.log('   - Detected empty rows');
        console.log('   - Identified missing required fields with specifics');
        console.log('   - Caught invalid characters and formats');
        console.log('   - Validated field lengths');
        console.log('   - Handled price parsing errors');
        
        // Clean up
        if (fs.existsSync('./test_validation_logic.csv')) {
          fs.unlinkSync('./test_validation_logic.csv');
        }
        
        resolve();
      });
  });
}

testValidationLogic();