tilbudgivern/analyze_csv_problems.js

const fs = require('fs');
const csv = require('csv-parser');

async function analyzeCSVProblems() {
  console.log('🔍 Analyzing CSV for problematic rows...');
  
  const filePath = './Bygma-Prisbog-DK-1000070-20241201-093000.csv';
  
  return new Promise((resolve, reject) => {
    let lineNumber = 0;
    const problems = [];
    const emptyRows = [];
    const missingRequiredFields = [];
    const invalidPrices = [];
    
    fs.createReadStream(filePath)
      .pipe(csv({
        separator: ';',
        headers: ['Varegrp', 'VareNr', 'Tekst', 'Enhed', 'BruttoPris', 'Opdat', 'Nettopris', 'DB-nr', 'EAN-nr', 'Std'],
        skipLinesWithError: false,
        strict: false
      }))
      .on('data', (row) => {
        lineNumber++;
        
        // Skip header
        if (lineNumber === 1) return;
        
        // Check for completely empty rows
        const nonEmptyFields = Object.values(row).filter(value => value && value.toString().trim() !== '').length;
        if (nonEmptyFields === 0) {
          emptyRows.push({ line: lineNumber, data: row });
          return;
        }
        
        // Check for rows with too few fields
        if (nonEmptyFields < 3) {
          problems.push({ 
            line: lineNumber, 
            type: 'TOO_FEW_FIELDS', 
            data: row,
            nonEmptyCount: nonEmptyFields
          });
          return;
        }
        
        // Check required fields
        if (!row.VareNr || !row.Tekst || !row.Enhed) {
          missingRequiredFields.push({
            line: lineNumber,
            missing: {
              VareNr: !row.VareNr,
              Tekst: !row.Tekst,
              Enhed: !row.Enhed
            },
            data: row
          });
        }
        
        // Check prices
        const bruttoPrice = parseFloat(row.BruttoPris);
        const nettoPrice = parseFloat(row.Nettopris);
        
        if (isNaN(bruttoPrice) || bruttoPrice <= 0) {
          invalidPrices.push({
            line: lineNumber,
            type: 'INVALID_BRUTTO',
            value: row.BruttoPris,
            data: row
          });
        }
        
        if (isNaN(nettoPrice) || nettoPrice <= 0) {
          invalidPrices.push({
            line: lineNumber,
            type: 'INVALID_NETTO',
            value: row.Nettopris,
            data: row
          });
        }
      })
      .on('end', () => {
        console.log('\n📊 CSV Analysis Results:');
        console.log(`   - Total rows processed: ${lineNumber}`);
        console.log(`   - Empty rows: ${emptyRows.length}`);
        console.log(`   - Rows with too few fields: ${problems.length}`);
        console.log(`   - Missing required fields: ${missingRequiredFields.length}`);
        console.log(`   - Invalid prices: ${invalidPrices.length}`);
        
        if (problems.length > 0) {
          console.log('\n⚠️  First 10 problematic rows:');
          problems.slice(0, 10).forEach((problem, index) => {
            console.log(`${index + 1}. Line ${problem.line}: ${problem.type} (${problem.nonEmptyCount} fields)`);
            console.log(`   Data: ${JSON.stringify(problem.data)}`);
          });
        }
        
        if (missingRequiredFields.length > 0) {
          console.log('\n⚠️  First 10 rows with missing fields:');
          missingRequiredFields.slice(0, 10).forEach((missing, index) => {
            console.log(`${index + 1}. Line ${missing.line}:`);
            console.log(`   Missing: ${Object.entries(missing.missing).filter(([k,v]) => v).map(([k,v]) => k).join(', ')}`);
            console.log(`   Data: ${JSON.stringify(missing.data)}`);
          });
        }
        
        if (invalidPrices.length > 0) {
          console.log('\n⚠️  First 10 invalid prices:');
          invalidPrices.slice(0, 10).forEach((price, index) => {
            console.log(`${index + 1}. Line ${price.line}: ${price.type} = "${price.value}"`);
          });
        }
        
        resolve();
      })
      .on('error', reject);
  });
}

analyzeCSVProblems().catch(console.error);