Add migration script to import JSON data into SQLite database
- Created a new script `migrate-to-sqlite.js` to migrate data from JSON files into a SQLite database. - Implemented functions to read JSON files safely and handle errors. - Established database schema with tables for armour, weapons, bestiary, and rules. - Added logic to insert data from JSON files into the corresponding database tables. - Included backup functionality for the existing database before migration. - Logged import totals and sample data from each table for verification. - Added a new script `check-shop-stats.js` to fetch and log item statistics from the database.
This commit is contained in:
@@ -1,51 +1,86 @@
|
||||
const { categorizedWeapons } = require('./parse-weapons');
|
||||
const { categorizedArmor } = require('./parse-armor');
|
||||
const comprehensiveWeapons = require('./comprehensive-weapons');
|
||||
const comprehensiveArmor = require('./comprehensive-armor');
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const Database = require('better-sqlite3')
|
||||
|
||||
// Combine all shop items into a single database
|
||||
const buildShopDatabase = () => {
|
||||
const shopItems = {
|
||||
// Use comprehensive weapons dataset
|
||||
rangedWeapons: comprehensiveWeapons.rangedWeapons,
|
||||
meleeWeapons: comprehensiveWeapons.meleeWeapons,
|
||||
grenades: comprehensiveWeapons.grenades,
|
||||
otherWeapons: comprehensiveWeapons.other,
|
||||
|
||||
// Use comprehensive armor dataset
|
||||
powerArmor: comprehensiveArmor.powerArmor,
|
||||
powerArmorHelms: comprehensiveArmor.powerArmorHelms,
|
||||
carapaceArmor: comprehensiveArmor.carapaceArmor,
|
||||
naturalArmor: comprehensiveArmor.naturalArmor,
|
||||
primitiveArmor: comprehensiveArmor.primitiveArmor,
|
||||
xenosArmor: comprehensiveArmor.xenosArmor,
|
||||
shields: comprehensiveArmor.shields,
|
||||
otherArmor: comprehensiveArmor.otherArmor
|
||||
};
|
||||
// Ensure source data files are available
|
||||
const weaponsPath = path.join(__dirname, '../database/public/deathwatch-weapons-comprehensive.json')
|
||||
const armourPath = path.join(__dirname, '../database/public/deathwatch-armor-comprehensive.json')
|
||||
let comprehensiveWeapons = { rangedWeapons: [], meleeWeapons: [], grenades: [], other: [] }
|
||||
let comprehensiveArmor = {}
|
||||
try { comprehensiveWeapons = JSON.parse(fs.readFileSync(weaponsPath, 'utf8')) } catch (e) { console.warn('Could not read weapons JSON:', e && e.message) }
|
||||
try { comprehensiveArmor = JSON.parse(fs.readFileSync(armourPath, 'utf8')) } catch (e) { console.warn('Could not read armour JSON:', e && e.message) }
|
||||
|
||||
// Add metadata
|
||||
const shopDatabase = {
|
||||
version: "3.0.0",
|
||||
lastUpdated: new Date().toISOString(),
|
||||
categories: Object.keys(shopItems),
|
||||
items: shopItems
|
||||
};
|
||||
const dbPath = path.join(__dirname, '..', 'database', 'sqlite', 'deathwatch.db')
|
||||
|
||||
return shopDatabase;
|
||||
};
|
||||
|
||||
// Write the database
|
||||
if (require.main === module) {
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// First ensure our source data is up to date
|
||||
require('./parse-weapons');
|
||||
require('./parse-armor');
|
||||
|
||||
// Then build and write the combined database
|
||||
const output = path.resolve('../public/deathwatch-armoury.json');
|
||||
const database = buildShopDatabase();
|
||||
fs.writeFileSync(output, JSON.stringify(database, null, 2), 'utf8');
|
||||
console.log('Wrote combined shop database to:', output);
|
||||
function safeRead(obj, key) {
|
||||
try { return obj[key] } catch (e) { return undefined }
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
if (!fs.existsSync(dbPath)) {
|
||||
console.error('Database not found at', dbPath)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const backup = dbPath + '.backup.' + Date.now()
|
||||
fs.copyFileSync(dbPath, backup)
|
||||
console.log('Backup DB created at', backup)
|
||||
|
||||
const db = new Database(dbPath)
|
||||
|
||||
// Ensure shop tables exist (idempotent)
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS shop_items (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
category TEXT NOT NULL,
|
||||
requisition_cost INTEGER NOT NULL DEFAULT 0,
|
||||
renown_requirement TEXT NOT NULL DEFAULT 'None',
|
||||
item_type TEXT NOT NULL,
|
||||
stats TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`)
|
||||
|
||||
const insert = db.prepare(`
|
||||
INSERT OR REPLACE INTO shop_items (name, category, requisition_cost, renown_requirement, item_type, stats, source)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
|
||||
const tx = db.transaction((items) => {
|
||||
for (const it of items) {
|
||||
const name = it.name || it.id || '(unnamed)'
|
||||
const category = it.category || 'Misc'
|
||||
const req = Number(it.req ?? it.cost ?? it.requisition_cost ?? 0) || 0
|
||||
const renown = it.renown || it.renown_level || 'None'
|
||||
const itemType = (it.stats && (it.stats.class || it.stats.type)) || safeRead(it, 'category') || 'gear'
|
||||
const stats = JSON.stringify(it.stats || {})
|
||||
const source = (it.stats && it.stats.source) || it.source || ''
|
||||
insert.run(name, category, req, renown, itemType, stats, source)
|
||||
}
|
||||
})
|
||||
|
||||
// Collect items from comprehensive datasets
|
||||
const allItems = []
|
||||
const weaponCats = ['rangedWeapons','meleeWeapons','grenades','other']
|
||||
for (const cat of weaponCats) {
|
||||
if (Array.isArray(comprehensiveWeapons[cat])) allItems.push(...comprehensiveWeapons[cat].map(it => ({ ...it, category: cat })))
|
||||
}
|
||||
|
||||
const armorCats = ['powerArmor','powerArmorHelms','carapaceArmor','naturalArmor','primitiveArmor','xenosArmor','shields','otherArmor']
|
||||
for (const cat of armorCats) {
|
||||
if (Array.isArray(comprehensiveArmor[cat])) allItems.push(...comprehensiveArmor[cat].map(it => ({ ...it, category: cat })))
|
||||
}
|
||||
|
||||
console.log('Inserting', allItems.length, 'shop items into DB...')
|
||||
try {
|
||||
tx(allItems)
|
||||
console.log('Inserted shop items successfully')
|
||||
} catch (e) {
|
||||
console.error('Failed to insert shop items:', e)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
11
scripts/check-shop-stats.js
Normal file
11
scripts/check-shop-stats.js
Normal file
@@ -0,0 +1,11 @@
|
||||
const shop = require('../database/shop-helpers');
|
||||
|
||||
shop.getAllItems().then(items => {
|
||||
console.log('items:', items.length);
|
||||
const s = items.find(i => i.id === 786) || items[0];
|
||||
console.log('sample id', s.id, 'stats type', typeof s.stats);
|
||||
console.log('stats sample', JSON.stringify(s.stats, null, 2).slice(0, 400));
|
||||
}).catch(err => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
166
scripts/migrate-to-sqlite.js
Normal file
166
scripts/migrate-to-sqlite.js
Normal file
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
|
||||
const dbPath = path.join(__dirname, '../database/sqlite/deathwatch.db');
|
||||
const backupPath = dbPath + '.backup.' + Date.now();
|
||||
|
||||
function safeReadJSON(p) {
|
||||
if (!fs.existsSync(p)) return null;
|
||||
try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch (e) { console.error('JSON parse error', p, e.message); return null; }
|
||||
}
|
||||
|
||||
console.log('=== MIGRATE JSON DATA INTO SQLITE ===');
|
||||
if (!fs.existsSync(dbPath)) {
|
||||
console.error('DB not found at', dbPath);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
fs.copyFileSync(dbPath, backupPath);
|
||||
console.log('Backup created:', backupPath);
|
||||
|
||||
const db = new Database(dbPath);
|
||||
|
||||
// Create consolidated tables
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS armour (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
req INTEGER DEFAULT 0,
|
||||
renown TEXT DEFAULT 'None',
|
||||
category TEXT,
|
||||
stats TEXT,
|
||||
source TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS weapons (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
req INTEGER DEFAULT 0,
|
||||
renown TEXT DEFAULT 'None',
|
||||
category TEXT,
|
||||
stats TEXT,
|
||||
source TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bestiary (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
book TEXT,
|
||||
page TEXT,
|
||||
pdf TEXT,
|
||||
stats TEXT,
|
||||
profile TEXT,
|
||||
snippet TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
rule_id TEXT UNIQUE,
|
||||
title TEXT,
|
||||
content TEXT,
|
||||
page INTEGER,
|
||||
source TEXT,
|
||||
source_abbr TEXT,
|
||||
category TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
const insertArmour = db.prepare(`INSERT OR IGNORE INTO armour (name, req, renown, category, stats, source) VALUES (?, ?, ?, ?, ?, ?)`);
|
||||
const insertWeapon = db.prepare(`INSERT OR IGNORE INTO weapons (name, req, renown, category, stats, source) VALUES (?, ?, ?, ?, ?, ?)`);
|
||||
const insertBestiary = db.prepare(`INSERT INTO bestiary (name, book, page, pdf, stats, profile, snippet) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
||||
const insertRule = db.prepare(`INSERT OR IGNORE INTO rules (rule_id, title, content, page, source, source_abbr, category) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
||||
|
||||
let totals = { armour:0, weapons:0, bestiary:0, rules:0 };
|
||||
|
||||
// Import armour
|
||||
const armourFile = path.join(__dirname, '../database/public/deathwatch-armor.json');
|
||||
const armourData = safeReadJSON(armourFile);
|
||||
if (armourData) {
|
||||
const categories = Object.keys(armourData);
|
||||
categories.forEach(cat => {
|
||||
const arr = armourData[cat];
|
||||
if (!Array.isArray(arr)) return;
|
||||
const insert = insertArmour;
|
||||
db.transaction(() => {
|
||||
for (const item of arr) {
|
||||
const stats = JSON.stringify(item.stats || {});
|
||||
const src = (item.stats && item.stats.source) || '';
|
||||
insert.run(item.name || '(unnamed)', item.req || 0, item.renown || 'None', item.category || cat, stats, src);
|
||||
totals.armour++;
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
// Import weapons
|
||||
const weaponsFile = path.join(__dirname, '../database/public/deathwatch-weapons-comprehensive.json');
|
||||
const weaponsData = safeReadJSON(weaponsFile);
|
||||
if (weaponsData) {
|
||||
// many weapon files use keys like rangedWeapons, meleeWeapons
|
||||
Object.keys(weaponsData).forEach(k => {
|
||||
const arr = weaponsData[k];
|
||||
if (!Array.isArray(arr)) return;
|
||||
db.transaction(() => {
|
||||
for (const w of arr) {
|
||||
const stats = JSON.stringify(w.stats || {});
|
||||
const src = (w.stats && w.stats.source) || '';
|
||||
insertWeapon.run(w.name || '(unnamed)', w.req || 0, w.renown || 'Any', w.category || k, stats, src);
|
||||
totals.weapons++;
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
// Import bestiary
|
||||
const bestiaryFile = path.join(__dirname, '../database/deathwatch-bestiary-extracted.json');
|
||||
const bestiaryData = safeReadJSON(bestiaryFile);
|
||||
if (bestiaryData && Array.isArray(bestiaryData.results)) {
|
||||
db.transaction(() => {
|
||||
for (const e of bestiaryData.results) {
|
||||
const stats = JSON.stringify(e.stats || {});
|
||||
const profile = JSON.stringify(e.profile || {});
|
||||
const name = e.bestiaryName || e.name || '(unnamed)';
|
||||
insertBestiary.run(name, e.book || '', e.page || '', e.pdf || '', stats, profile, e.stats && e.stats.snippet ? e.stats.snippet : '');
|
||||
totals.bestiary++;
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
// Import rules
|
||||
const rulesFile = path.join(__dirname, '../database/rules/rules-database.json');
|
||||
const rulesData = safeReadJSON(rulesFile);
|
||||
if (rulesData && Array.isArray(rulesData.rules)) {
|
||||
db.transaction(() => {
|
||||
for (const r of rulesData.rules) {
|
||||
insertRule.run(r.id || null, r.title || '', r.content || '', r.page || null, r.source || '', r.sourceAbbr || '', r.category || 'general');
|
||||
totals.rules++;
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
console.log('Import totals:', totals);
|
||||
|
||||
// Show row counts from DB for verification
|
||||
const counts = {
|
||||
armour: db.prepare('SELECT COUNT(*) as c FROM armour').get().c,
|
||||
weapons: db.prepare('SELECT COUNT(*) as c FROM weapons').get().c,
|
||||
bestiary: db.prepare('SELECT COUNT(*) as c FROM bestiary').get().c,
|
||||
rules: db.prepare('SELECT COUNT(*) as c FROM rules').get().c
|
||||
};
|
||||
|
||||
console.log('DB row counts:', counts);
|
||||
|
||||
// Print a small sample from each table
|
||||
console.log('\nSample armour:', db.prepare('SELECT name, category, stats FROM armour LIMIT 3').all());
|
||||
console.log('\nSample weapons:', db.prepare('SELECT name, category, stats FROM weapons LIMIT 3').all());
|
||||
console.log('\nSample bestiary:', db.prepare('SELECT name, book, snippet FROM bestiary LIMIT 3').all());
|
||||
console.log('\nSample rules:', db.prepare('SELECT rule_id, title FROM rules LIMIT 3').all());
|
||||
|
||||
db.close();
|
||||
console.log('\nMigration complete. DB backed up at', backupPath);
|
||||
Reference in New Issue
Block a user