- Created setup scripts for advanced database, pricing database, and quote items. - Added a test script for OCR functionality, including PDF text extraction and price data extraction. - Included test files for various scenarios: poor categorization, material list, and text with noise.
223 lines
6.9 KiB
JavaScript
Executable File
223 lines
6.9 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
||
|
||
/**
|
||
* Database Cleanup CLI Tool
|
||
* Usage: node cleanup-cli.js [command] [options]
|
||
*/
|
||
|
||
const databaseCleanupService = require('./src/services/databaseCleanupService');
|
||
const logger = require('./src/utils/logger');
|
||
|
||
const commands = {
|
||
stats: 'Show database statistics',
|
||
preview: 'Preview what cleanup would do (dry run)',
|
||
cleanup: 'Run material cleanup',
|
||
'remove-irrelevant': 'Remove materials marked as irrelevant',
|
||
help: 'Show this help message'
|
||
};
|
||
|
||
async function showStats() {
|
||
try {
|
||
const stats = await databaseCleanupService.getMaterialCategoryStats();
|
||
|
||
console.log('\n📊 DATABASE STATISTICS');
|
||
console.log('=====================');
|
||
console.log(`Total materials: ${stats.totalMaterials}`);
|
||
console.log(`Problem materials: ${stats.problemMaterials}`);
|
||
console.log(`Cleanup recommended: ${stats.cleanupRecommended ? 'YES' : 'NO'}`);
|
||
|
||
console.log('\n📋 CATEGORY BREAKDOWN:');
|
||
stats.categoryBreakdown.forEach(cat => {
|
||
console.log(` ${cat.category}: ${cat.count} items (${cat.with_subcategory} with subcategory)`);
|
||
});
|
||
|
||
if (stats.cleanupRecommended) {
|
||
console.log('\n💡 Recommendation: Run "node cleanup-cli.js preview" to see what can be cleaned up');
|
||
}
|
||
} catch (error) {
|
||
console.error('❌ Error getting stats:', error.message);
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
async function showPreview(limit = 10) {
|
||
try {
|
||
console.log(`\n🔍 CLEANUP PREVIEW (limit: ${limit})`);
|
||
console.log('=====================================');
|
||
|
||
const result = await databaseCleanupService.previewCleanup(limit);
|
||
|
||
if (result.processed === 0) {
|
||
console.log('✅ No materials need cleanup!');
|
||
return;
|
||
}
|
||
|
||
console.log(`Materials that would be processed: ${result.processed}`);
|
||
console.log(`Materials that would be updated: ${result.updated}`);
|
||
|
||
if (result.details && result.details.length > 0) {
|
||
console.log('\n📝 CHANGES THAT WOULD BE MADE:');
|
||
result.details.forEach(detail => {
|
||
const action = detail.isRelevant ? 'UPDATE' : 'DELETE';
|
||
console.log(` [${action}] ID ${detail.id}: ${detail.oldCategory}/${detail.oldSubcategory} → ${detail.newCategory}/${detail.newSubcategory}`);
|
||
});
|
||
}
|
||
|
||
if (result.errors.length > 0) {
|
||
console.log('\n⚠️ ERRORS:');
|
||
result.errors.forEach(error => {
|
||
console.log(` ${error.materialName}: ${error.error}`);
|
||
});
|
||
}
|
||
|
||
console.log('\n💡 To apply changes, run: node cleanup-cli.js cleanup');
|
||
} catch (error) {
|
||
console.error('❌ Error in preview:', error.message);
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
async function runCleanup(limit = 50, onlyInvalid = true) {
|
||
try {
|
||
console.log(`\n🧹 RUNNING CLEANUP (limit: ${limit}, onlyInvalid: ${onlyInvalid})`);
|
||
console.log('===============================================');
|
||
|
||
const result = await databaseCleanupService.cleanupMaterials({
|
||
limit,
|
||
onlyInvalid,
|
||
dryRun: false
|
||
});
|
||
|
||
console.log(`✅ Cleanup completed!`);
|
||
console.log(`Materials processed: ${result.processed}`);
|
||
console.log(`Materials updated: ${result.updated}`);
|
||
console.log(`Errors: ${result.errors.length}`);
|
||
|
||
if (result.details && result.details.length > 0) {
|
||
console.log('\n📝 CHANGES MADE:');
|
||
result.details.forEach(detail => {
|
||
const action = detail.isRelevant ? 'UPDATED' : 'MARKED FOR DELETION';
|
||
console.log(` [${action}] ID ${detail.id}: ${detail.oldCategory}/${detail.oldSubcategory} → ${detail.newCategory}/${detail.newSubcategory}`);
|
||
});
|
||
}
|
||
|
||
if (result.errors.length > 0) {
|
||
console.log('\n⚠️ ERRORS:');
|
||
result.errors.forEach(error => {
|
||
console.log(` ${error.materialName}: ${error.error}`);
|
||
});
|
||
}
|
||
|
||
console.log('\n💡 Run "node cleanup-cli.js remove-irrelevant" to remove materials marked for deletion');
|
||
} catch (error) {
|
||
console.error('❌ Error in cleanup:', error.message);
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
async function removeIrrelevant(confirm = false) {
|
||
try {
|
||
console.log('\n🗑️ REMOVING IRRELEVANT MATERIALS');
|
||
console.log('=================================');
|
||
|
||
const result = await databaseCleanupService.removeIrrelevantMaterials(confirm);
|
||
|
||
if (result.action === 'preview') {
|
||
console.log(`Found ${result.materialsToDelete.length} materials marked for deletion:`);
|
||
result.materialsToDelete.forEach(material => {
|
||
console.log(` - ID ${material.id}: ${material.name}`);
|
||
});
|
||
|
||
if (result.materialsToDelete.length > 0) {
|
||
console.log('\n💡 To confirm deletion, run: node cleanup-cli.js remove-irrelevant --confirm');
|
||
}
|
||
} else {
|
||
console.log(`✅ Deleted ${result.deletedCount} irrelevant materials`);
|
||
}
|
||
} catch (error) {
|
||
console.error('❌ Error removing irrelevant materials:', error.message);
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
function showHelp() {
|
||
console.log('\n🛠️ DATABASE CLEANUP CLI TOOL');
|
||
console.log('============================');
|
||
console.log('\nUsage: node cleanup-cli.js [command] [options]');
|
||
console.log('\nCommands:');
|
||
|
||
Object.entries(commands).forEach(([cmd, desc]) => {
|
||
console.log(` ${cmd.padEnd(20)} ${desc}`);
|
||
});
|
||
|
||
console.log('\nExamples:');
|
||
console.log(' node cleanup-cli.js stats');
|
||
console.log(' node cleanup-cli.js preview --limit 20');
|
||
console.log(' node cleanup-cli.js cleanup --limit 100 --all');
|
||
console.log(' node cleanup-cli.js remove-irrelevant --confirm');
|
||
console.log('\nOptions:');
|
||
console.log(' --limit N Process maximum N materials (default: 50 for cleanup, 10 for preview)');
|
||
console.log(' --all Process all materials, not just invalid ones');
|
||
console.log(' --confirm Confirm destructive operations');
|
||
}
|
||
|
||
async function main() {
|
||
const args = process.argv.slice(2);
|
||
const command = args[0] || 'help';
|
||
|
||
// Parse options
|
||
const options = {
|
||
limit: null,
|
||
all: args.includes('--all'),
|
||
confirm: args.includes('--confirm')
|
||
};
|
||
|
||
const limitIndex = args.indexOf('--limit');
|
||
if (limitIndex !== -1 && args[limitIndex + 1]) {
|
||
options.limit = parseInt(args[limitIndex + 1]);
|
||
}
|
||
|
||
console.log('🚀 Starting Database Cleanup Tool...');
|
||
|
||
try {
|
||
switch (command) {
|
||
case 'stats':
|
||
await showStats();
|
||
break;
|
||
|
||
case 'preview':
|
||
await showPreview(options.limit || 10);
|
||
break;
|
||
|
||
case 'cleanup':
|
||
await runCleanup(
|
||
options.limit || 50,
|
||
!options.all // onlyInvalid = !all
|
||
);
|
||
break;
|
||
|
||
case 'remove-irrelevant':
|
||
await removeIrrelevant(options.confirm);
|
||
break;
|
||
|
||
case 'help':
|
||
default:
|
||
showHelp();
|
||
break;
|
||
}
|
||
|
||
console.log('\n✨ Done!');
|
||
process.exit(0);
|
||
} catch (error) {
|
||
console.error('\n💥 Fatal error:', error.message);
|
||
process.exit(1);
|
||
}
|
||
}
|
||
|
||
// Run if called directly
|
||
if (require.main === module) {
|
||
main();
|
||
}
|
||
|
||
module.exports = { main };
|