alex/dwroller
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add migration script to import JSON data into SQLite database

Browse Source 
- Created a new script `migrate-to-sqlite.js` to migrate data from JSON files into a SQLite database.
- Implemented functions to read JSON files safely and handle errors.
- Established database schema with tables for armour, weapons, bestiary, and rules.
- Added logic to insert data from JSON files into the corresponding database tables.
- Included backup functionality for the existing database before migration.
- Logged import totals and sample data from each table for verification.
- Added a new script `check-shop-stats.js` to fetch and log item statistics from the database.
This commit is contained in:
Alex
2025-08-17 19:14:29 +02:00
parent 4c1b19fb8c
commit 9d53410aab
16 changed files with 496 additions and 374 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
database/backend.log
View File

@@ -9774,3 +9774,6 @@ Connected to MongoDB
[2025-08-17T16:05:12.595Z] DB: getSession - start 37bc66f655c6bd40a2cce1f31aa73489495ee3b7555af82c67254a0d98c0c706
[2025-08-17T16:05:12.595Z] DB: getSession - ok 37bc66f655c6bd40a2cce1f31aa73489495ee3b7555af82c67254a0d98c0c706
[2025-08-17T16:05:12.595Z] SESSION: Validate success 37bc66f655c6bd40a2cce1f31aa73489495ee3b7555af82c67254a0d98c0c706 gm
[2025-08-17T17:12:14.022Z] DB: getSession - start 37bc66f655c6bd40a2cce1f31aa73489495ee3b7555af82c67254a0d98c0c706
[2025-08-17T17:12:14.023Z] DB: getSession - ok 37bc66f655c6bd40a2cce1f31aa73489495ee3b7555af82c67254a0d98c0c706
[2025-08-17T17:12:14.023Z] SESSION: Validate success 37bc66f655c6bd40a2cce1f31aa73489495ee3b7555af82c67254a0d98c0c706 gm

41
database/routes/bestiaryRoutes.js
View File

@@ -1,30 +1,31 @@
const express = require('express');
const fs = require('fs');
const path = require('path');
const { db } = require('../sqlite-db');
const router = express.Router();
const BESTIARY_PATH = path.join(__dirname, '../../database/deathwatch-bestiary-extracted.json');
let bestiaryData = null;
let lastLoaded = 0;
// Read bestiary from sqlite table `bestiary`
function loadBestiaryData() {
const now = Date.now();
// Cache for 5 minutes
if (bestiaryData && (now - lastLoaded) < 5 * 60 * 1000) {
return bestiaryData;
}
try {
console.log('Loading bestiary data from:', BESTIARY_PATH);
const data = fs.readFileSync(BESTIARY_PATH, 'utf8');
const parsed = JSON.parse(data);
bestiaryData = parsed.results || [];
lastLoaded = now;
console.log(`Loaded ${bestiaryData.length} bestiary entries`);
return bestiaryData;
} catch (error) {
console.error('Failed to load bestiary data:', error);
const rows = db.prepare('SELECT id,name,book,page,pdf,stats,profile,snippet FROM bestiary ORDER BY name').all();
return rows.map(r => {
let stats = {};
try { stats = JSON.parse(r.stats || '{}'); } catch(e){}
let profile = {};
try { profile = JSON.parse(r.profile || '{}'); } catch(e){}
return {
_id: r.id,
bestiaryName: r.name,
book: r.book,
page: r.page,
pdf: r.pdf,
stats: stats,
profile: profile,
snippet: r.snippet
};
});
} catch (e) {
console.error('Failed to load bestiary from sqlite:', e);
return [];
}
}

20
database/routes/playerRoutes-sqlite.js
View File

@@ -1,10 +1,9 @@
const express = require('express');
const { playerHelpers, sessionHelpers } = require('../sqlite-db');
const { playerHelpers, sessionHelpers, db } = require('../sqlite-db');
const fs = require('fs');
const path = require('path');
// Load shop data
const shopData = JSON.parse(fs.readFileSync(path.join(__dirname, '../../public/deathwatch-armoury.json'), 'utf8'));
// shop data will be read from sqlite `shop_items` table when needed
// Simple file logger
function logToFile(...args) {
@@ -21,11 +20,16 @@ const router = express.Router();
router.get('/shop', (req, res) => {
try {
console.log('Shop endpoint hit');
const filePath = path.join(__dirname, '../../public/deathwatch-armoury.json');
console.log('Looking for shop data at:', filePath);
const shopData = JSON.parse(fs.readFileSync(filePath, 'utf8'));
console.log('Shop data loaded:', Object.keys(shopData));
res.json(shopData);
// Query sqlite shop_items and return grouped by category
const items = db.prepare('SELECT id, name, category, requisition_cost as req, renown_requirement as renown, item_type, stats, source FROM shop_items ORDER BY category, name').all();
const grouped = items.reduce((acc, it) => {
acc[it.category] = acc[it.category] || [];
let stats = {};
try { stats = JSON.parse(it.stats || '{}'); } catch(e){}
acc[it.category].push({ id: it.id, name: it.name, req: it.req, renown: it.renown, itemType: it.item_type, stats, source: it.source });
return acc;
}, {});
res.json(grouped);
} catch (error) {
console.error('Shop error:', error);
logToFile('API: Failed to get shop data', error);

244
database/routes/rulesRoutes.js
View File

@@ -1,61 +1,37 @@
const express = require('express');
const fs = require('fs');
const path = require('path');
const { db } = require('../sqlite-db');
const router = express.Router();
console.log('Loading rules routes...');
console.log('Rules routes registered (sqlite-backed)');
// Load rules database
let rulesDatabase = null;
let searchIndex = null;
function loadRulesDatabase() {
// We will query sqlite `rules` table on demand; helper to fetch all rules
function getAllRules() {
try {
const dbPath = path.join(__dirname, '../rules/rules-database.json');
if (fs.existsSync(dbPath)) {
const data = JSON.parse(fs.readFileSync(dbPath, 'utf8'));
rulesDatabase = data.rules;
searchIndex = data.searchIndex;
console.log(`Loaded ${rulesDatabase.length} rules from database`);
return true;
}
console.warn('Rules database not found, using empty database');
rulesDatabase = [];
searchIndex = {};
return false;
} catch (error) {
console.error('Failed to load rules database:', error);
rulesDatabase = [];
searchIndex = {};
return false;
const rows = db.prepare('SELECT id, rule_id, title, content, page, source, source_abbr, category FROM rules ORDER BY id').all();
return rows.map(r => ({ id: r.id, rule_id: r.rule_id, title: r.title, content: r.content, page: r.page, source: r.source, sourceAbbr: r.source_abbr, category: r.category }));
} catch (e) {
console.error('Failed to read rules from sqlite:', e);
return [];
}
}
function getRuleById(ruleId) {
try {
const row = db.prepare('SELECT rule_id as id, title, content, page, source, source_abbr as sourceAbbr, category FROM rules WHERE rule_id = ?').get(ruleId);
return row || null;
} catch (e) {
console.error('Failed to read rule by id:', e);
return null;
}
}
// Load the database on startup
loadRulesDatabase();
// Get all rule categories
router.get('/categories', (req, res) => {
try {
if (!rulesDatabase) {
return res.json([
{ id: 'all', name: 'All Rules' },
{ id: 'combat', name: 'Combat' },
{ id: 'weapons', name: 'Weapons' },
{ id: 'armor', name: 'Armor' },
{ id: 'skills', name: 'Skills' },
{ id: 'talents', name: 'Talents' },
{ id: 'psychic', name: 'Psychic Powers' },
{ id: 'equipment', name: 'Equipment' }
]);
}
const categories = [...new Set(rulesDatabase.map(rule => rule.category))];
const categoryList = categories.map(cat => ({
id: cat,
name: cat.charAt(0).toUpperCase() + cat.slice(1)
}));
const rows = getAllRules();
const categories = [...new Set(rows.map(r => r.category).filter(Boolean))];
const categoryList = categories.map(cat => ({ id: cat, name: cat.charAt(0).toUpperCase() + cat.slice(1) }));
res.json([{ id: 'all', name: 'All Rules' }, ...categoryList]);
} catch (error) {
console.error('Categories error:', error);
@@ -63,97 +39,26 @@ router.get('/categories', (req, res) => {
}
});
// Search rules
// Search rules (sqlite-backed)
router.get('/search', (req, res) => {
console.log('Search route hit! Query:', req.query);
try {
const { q: query, category, limit = 20 } = req.query;
if (!query || !query.trim()) {
return res.json([]);
}
if (!rulesDatabase || rulesDatabase.length === 0) {
return res.json([
{
id: 'no_database',
title: 'Rules Database Not Available',
content: 'The rules database has not been loaded. Run the extract-rules.js script to populate the database from the PDF files.',
category: 'system',
page: null,
source: 'System'
}
]);
}
const searchTerms = query.toLowerCase().split(/\s+/).filter(term => term.length > 2);
// scoring helper - returns Map(index -> score)
const scoreResults = (applyCategoryFilter) => {
const scores = new Map();
rulesDatabase.forEach((rule, index) => {
let score = 0;
const titleLower = (rule.title || '').toLowerCase();
const contentLower = (rule.content || '').toLowerCase();
if (titleLower === query.toLowerCase()) score += 100;
if (titleLower.includes(query.toLowerCase())) score += 50;
searchTerms.forEach(term => {
if (titleLower.includes(term)) score += 20;
if (contentLower.includes(term)) score += 5;
if (searchIndex && searchIndex[term] && searchIndex[term].includes(index)) score += 3;
});
if (applyCategoryFilter && category && category !== 'all' && rule.category !== category) {
// skip when category filtering is requested
score = 0;
}
if (score > 0) scores.set(index, score);
});
return scores;
};
if (!query || !query.trim()) return res.json([]);
const limitInt = Math.max(1, parseInt(limit) || 20);
const term = `%${query}%`;
// Primary: category-filtered results (if category requested)
const primaryScores = scoreResults(true);
const primarySorted = Array.from(primaryScores.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, limitInt)
.map(([index]) => index);
// Secondary: best matches ignoring category (fallback)
const secondaryScores = scoreResults(false);
const secondarySorted = Array.from(secondaryScores.entries())
.sort((a, b) => b[1] - a[1])
.map(([index]) => index)
.filter(idx => !primarySorted.includes(idx))
.slice(0, Math.max(0, limitInt - primarySorted.length));
// If a category is requested but yields no primary results, fall back to best cross-category matches
let combinedIndices;
let rows;
if (category && category !== 'all') {
if (primarySorted.length === 0 && secondarySorted.length > 0) {
combinedIndices = secondarySorted;
} else {
combinedIndices = [...primarySorted, ...secondarySorted];
rows = db.prepare('SELECT rule_id as id, title, content, page, source, source_abbr as sourceAbbr, category FROM rules WHERE (title LIKE ? OR content LIKE ?) AND category = ? ORDER BY id LIMIT ?').all(term, term, category, limitInt);
if (!rows || rows.length === 0) {
rows = db.prepare('SELECT rule_id as id, title, content, page, source, source_abbr as sourceAbbr, category FROM rules WHERE (title LIKE ? OR content LIKE ?) ORDER BY id LIMIT ?').all(term, term, limitInt);
}
} else {
combinedIndices = Array.from(secondaryScores.entries()).sort((a,b)=>b[1]-a[1]).slice(0, limitInt).map(([i])=>i);
rows = db.prepare('SELECT rule_id as id, title, content, page, source, source_abbr as sourceAbbr, category FROM rules WHERE (title LIKE ? OR content LIKE ?) ORDER BY id LIMIT ?').all(term, term, limitInt);
}
const results = combinedIndices.slice(0, limitInt).map(index => ({
...rulesDatabase[index],
content: rulesDatabase[index].content && rulesDatabase[index].content.length > 300
? rulesDatabase[index].content.substring(0, 300) + '...'
: rulesDatabase[index].content
}));
console.log(`Rules search: "${query}" (category: ${category || 'all'}) - ${results.length} results (primary ${primarySorted.length}, secondary ${secondarySorted.length})`);
const results = (rows || []).map(r => ({ ...r, content: r.content && r.content.length > 300 ? r.content.substring(0,300) + '...' : r.content }));
res.json(results);
} catch (error) {
console.error('Search error:', error);
res.status(500).json({ error: 'Search failed' });
@@ -164,16 +69,8 @@ router.get('/search', (req, res) => {
router.get('/rule/:id', (req, res) => {
try {
const { id } = req.params;
if (!rulesDatabase) {
return res.status(404).json({ error: 'Rules database not available' });
}
const rule = rulesDatabase.find(r => r.id === id);
if (!rule) {
return res.status(404).json({ error: 'Rule not found' });
}
const rule = getRuleById(id);
if (!rule) return res.status(404).json({ error: 'Rule not found' });
res.json(rule);
} catch (error) {
console.error('Get rule error:', error);
@@ -185,33 +82,14 @@ router.get('/rule/:id', (req, res) => {
router.get('/random', (req, res) => {
try {
const { count = 5, category } = req.query;
if (!rulesDatabase || rulesDatabase.length === 0) {
return res.json([]);
}
let eligibleRules = rulesDatabase;
const max = Math.max(1, parseInt(count) || 5);
let rows;
if (category && category !== 'all') {
eligibleRules = rulesDatabase.filter(rule => rule.category === category);
rows = db.prepare('SELECT rule_id as id, title, content, page, source, source_abbr as sourceAbbr, category FROM rules WHERE category = ? ORDER BY RANDOM() LIMIT ?').all(category, max);
} else {
rows = db.prepare('SELECT rule_id as id, title, content, page, source, source_abbr as sourceAbbr, category FROM rules ORDER BY RANDOM() LIMIT ?').all(max);
}
const randomRules = [];
const maxCount = Math.min(parseInt(count), eligibleRules.length);
const usedIndices = new Set();
while (randomRules.length < maxCount && usedIndices.size < eligibleRules.length) {
const randomIndex = Math.floor(Math.random() * eligibleRules.length);
if (!usedIndices.has(randomIndex)) {
usedIndices.add(randomIndex);
randomRules.push({
...eligibleRules[randomIndex],
content: eligibleRules[randomIndex].content.length > 200
? eligibleRules[randomIndex].content.substring(0, 200) + '...'
: eligibleRules[randomIndex].content
});
}
}
const randomRules = (rows || []).map(r => ({ ...r, content: r.content && r.content.length > 200 ? r.content.substring(0,200) + '...' : r.content }));
res.json(randomRules);
} catch (error) {
console.error('Random rules error:', error);
@@ -222,33 +100,10 @@ router.get('/random', (req, res) => {
// Get rules statistics
router.get('/stats', (req, res) => {
try {
if (!rulesDatabase) {
return res.json({
totalRules: 0,
categories: [],
sources: [],
searchTerms: 0
});
}
const categories = [...new Set(rulesDatabase.map(rule => rule.category))];
const sources = [...new Set(rulesDatabase.map(rule => rule.source))];
const categoryStats = categories.map(cat => ({
category: cat,
count: rulesDatabase.filter(rule => rule.category === cat).length
}));
res.json({
totalRules: rulesDatabase.length,
categories: categoryStats,
sources: sources.map(source => ({
source,
count: rulesDatabase.filter(rule => rule.source === source).length
})),
searchTerms: Object.keys(searchIndex).length
});
const totalRules = db.prepare('SELECT COUNT(*) as c FROM rules').get().c;
const categories = db.prepare('SELECT category, COUNT(*) as c FROM rules GROUP BY category').all();
const sources = db.prepare('SELECT source, COUNT(*) as c FROM rules GROUP BY source').all();
res.json({ totalRules, categories, sources, searchTerms: 0 });
} catch (error) {
console.error('Stats error:', error);
res.status(500).json({ error: 'Failed to get stats' });
@@ -258,19 +113,10 @@ router.get('/stats', (req, res) => {
// Reload the rules database (admin only)
router.post('/reload', (req, res) => {
try {
// Check for GM secret
const gmSecret = req.headers['x-gm-secret'];
if (gmSecret !== 'bongo') {
return res.status(403).json({ error: 'Unauthorized' });
}
const success = loadRulesDatabase();
res.json({
success,
totalRules: rulesDatabase ? rulesDatabase.length : 0,
message: success ? 'Rules database reloaded successfully' : 'Failed to reload rules database'
});
if (gmSecret !== 'bongo') return res.status(403).json({ error: 'Unauthorized' });
const totalRules = db.prepare('SELECT COUNT(*) as c FROM rules').get().c;
res.json({ success: true, totalRules, message: 'Rules are sqlite-backed; no reload necessary' });
} catch (error) {
console.error('Reload error:', error);
res.status(500).json({ error: 'Failed to reload database' });

21
database/routes/weaponsRoutes.js Normal file
View File

@@ -0,0 +1,21 @@
const express = require('express')
const router = express.Router()
const { db } = require('../sqlite-db')
// Return weapons in a normalized shape
router.get('/', (req, res) => {
try {
const rows = db.prepare('SELECT id,name,category,stats,source FROM weapons ORDER BY name').all()
const parsed = rows.map(r => {
let stats = {}
try { stats = JSON.parse(r.stats || '{}') } catch (e) {}
return { id: r.id, name: r.name, category: r.category, stats: stats, source: r.source }
})
res.json(parsed)
} catch (e) {
console.error('Failed to load weapons from sqlite:', e)
res.status(500).json({ error: 'Failed to get weapons' })
}
})
module.exports = router

4
database/server.js
View File

@@ -8,6 +8,7 @@ const sessionRoutes = require('./routes/sessionRoutes-sqlite');
const shopRoutes = require('./routes/shopRoutes');
const rulesRoutes = require('./routes/rulesRoutes');
const bestiaryRoutes = require('./routes/bestiaryRoutes');
const weaponsRoutes = require('./routes/weaponsRoutes');
// const rulesRoutes = require('./routes/rulesRoutes-simple');
console.log('Routes loaded:', {
@@ -143,6 +144,9 @@ console.log('Mounted /api/rules');
console.log('Mounting route: /api/bestiary');
app.use('/api/bestiary', bestiaryRoutes);
console.log('Mounted /api/bestiary');
console.log('Mounting route: /api/weapons');
app.use('/api/weapons', weaponsRoutes);
console.log('Mounted /api/weapons');
// Start Server
app.listen(PORT, '0.0.0.0', () => {

39
database/shop-helpers.js
View File

@@ -14,7 +14,16 @@ const shopHelpers = {
return new Promise((resolve, reject) => {
db.all('SELECT * FROM shop_items ORDER BY category, name', [], (err, rows) => {
if (err) return reject(err);
resolve(rows);
// Parse stats JSON string into object when possible
const parsed = rows.map(r => {
try {
return Object.assign({}, r, { stats: r.stats ? JSON.parse(r.stats) : r.stats });
} catch (e) {
// if parsing fails, leave as-is
return r;
}
});
resolve(parsed);
});
});
},
@@ -24,7 +33,14 @@ const shopHelpers = {
return new Promise((resolve, reject) => {
db.all('SELECT * FROM shop_items WHERE category = ? ORDER BY name', [category], (err, rows) => {
if (err) return reject(err);
resolve(rows);
const parsed = rows.map(r => {
try {
return Object.assign({}, r, { stats: r.stats ? JSON.parse(r.stats) : r.stats });
} catch (e) {
return r;
}
});
resolve(parsed);
});
});
},
@@ -42,7 +58,14 @@ const shopHelpers = {
ORDER BY si.category, si.name
`, [playerId], (err, rows) => {
if (err) return reject(err);
resolve(rows);
const parsed = rows.map(r => {
try {
return Object.assign({}, r, { stats: r.stats ? JSON.parse(r.stats) : r.stats });
} catch (e) {
return r;
}
});
resolve(parsed);
});
});
},
@@ -125,7 +148,15 @@ const shopHelpers = {
ORDER BY t.transaction_date DESC
`, [playerId], (err, rows) => {
if (err) return reject(err);
resolve(rows);
// transactions don't include full item stats, but parse if present
const parsed = rows.map(r => {
try {
return Object.assign({}, r, { stats: r.stats ? JSON.parse(r.stats) : r.stats });
} catch (e) {
return r;
}
});
resolve(parsed);
});
});
}

BIN
database/sqlite/deathwatch.db
View File

Binary file not shown.

BIN
database/sqlite/deathwatch.db.backup.1755447418502 Normal file
View File

Binary file not shown.

BIN
database/sqlite/deathwatch.db.backup.1755449835460 Normal file
View File

Binary file not shown.

129
scripts/build-shop-db.js
View File

@@ -1,51 +1,86 @@
const { categorizedWeapons } = require('./parse-weapons');
const { categorizedArmor } = require('./parse-armor');
const comprehensiveWeapons = require('./comprehensive-weapons');
const comprehensiveArmor = require('./comprehensive-armor');
const fs = require('fs')
const path = require('path')
const Database = require('better-sqlite3')
// Combine all shop items into a single database
const buildShopDatabase = () => {
const shopItems = {
// Use comprehensive weapons dataset
rangedWeapons: comprehensiveWeapons.rangedWeapons,
meleeWeapons: comprehensiveWeapons.meleeWeapons,
grenades: comprehensiveWeapons.grenades,
otherWeapons: comprehensiveWeapons.other,
// Use comprehensive armor dataset
powerArmor: comprehensiveArmor.powerArmor,
powerArmorHelms: comprehensiveArmor.powerArmorHelms,
carapaceArmor: comprehensiveArmor.carapaceArmor,
naturalArmor: comprehensiveArmor.naturalArmor,
primitiveArmor: comprehensiveArmor.primitiveArmor,
xenosArmor: comprehensiveArmor.xenosArmor,
shields: comprehensiveArmor.shields,
otherArmor: comprehensiveArmor.otherArmor
};
// Ensure source data files are available
const weaponsPath = path.join(__dirname, '../database/public/deathwatch-weapons-comprehensive.json')
const armourPath = path.join(__dirname, '../database/public/deathwatch-armor-comprehensive.json')
let comprehensiveWeapons = { rangedWeapons: [], meleeWeapons: [], grenades: [], other: [] }
let comprehensiveArmor = {}
try { comprehensiveWeapons = JSON.parse(fs.readFileSync(weaponsPath, 'utf8')) } catch (e) { console.warn('Could not read weapons JSON:', e && e.message) }
try { comprehensiveArmor = JSON.parse(fs.readFileSync(armourPath, 'utf8')) } catch (e) { console.warn('Could not read armour JSON:', e && e.message) }
// Add metadata
const shopDatabase = {
version: "3.0.0",
lastUpdated: new Date().toISOString(),
categories: Object.keys(shopItems),
items: shopItems
};
const dbPath = path.join(__dirname, '..', 'database', 'sqlite', 'deathwatch.db')
return shopDatabase;
};
// Write the database
if (require.main === module) {
const fs = require('fs');
const path = require('path');
// First ensure our source data is up to date
require('./parse-weapons');
require('./parse-armor');
// Then build and write the combined database
const output = path.resolve('../public/deathwatch-armoury.json');
const database = buildShopDatabase();
fs.writeFileSync(output, JSON.stringify(database, null, 2), 'utf8');
console.log('Wrote combined shop database to:', output);
function safeRead(obj, key) {
try { return obj[key] } catch (e) { return undefined }
}
if (require.main === module) {
if (!fs.existsSync(dbPath)) {
console.error('Database not found at', dbPath)
process.exit(1)
}
const backup = dbPath + '.backup.' + Date.now()
fs.copyFileSync(dbPath, backup)
console.log('Backup DB created at', backup)
const db = new Database(dbPath)
// Ensure shop tables exist (idempotent)
db.exec(`
CREATE TABLE IF NOT EXISTS shop_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
category TEXT NOT NULL,
requisition_cost INTEGER NOT NULL DEFAULT 0,
renown_requirement TEXT NOT NULL DEFAULT 'None',
item_type TEXT NOT NULL,
stats TEXT NOT NULL,
source TEXT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
`)
const insert = db.prepare(`
INSERT OR REPLACE INTO shop_items (name, category, requisition_cost, renown_requirement, item_type, stats, source)
VALUES (?, ?, ?, ?, ?, ?, ?)
`)
const tx = db.transaction((items) => {
for (const it of items) {
const name = it.name || it.id || '(unnamed)'
const category = it.category || 'Misc'
const req = Number(it.req ?? it.cost ?? it.requisition_cost ?? 0) || 0
const renown = it.renown || it.renown_level || 'None'
const itemType = (it.stats && (it.stats.class || it.stats.type)) || safeRead(it, 'category') || 'gear'
const stats = JSON.stringify(it.stats || {})
const source = (it.stats && it.stats.source) || it.source || ''
insert.run(name, category, req, renown, itemType, stats, source)
}
})
// Collect items from comprehensive datasets
const allItems = []
const weaponCats = ['rangedWeapons','meleeWeapons','grenades','other']
for (const cat of weaponCats) {
if (Array.isArray(comprehensiveWeapons[cat])) allItems.push(...comprehensiveWeapons[cat].map(it => ({ ...it, category: cat })))
}
const armorCats = ['powerArmor','powerArmorHelms','carapaceArmor','naturalArmor','primitiveArmor','xenosArmor','shields','otherArmor']
for (const cat of armorCats) {
if (Array.isArray(comprehensiveArmor[cat])) allItems.push(...comprehensiveArmor[cat].map(it => ({ ...it, category: cat })))
}
console.log('Inserting', allItems.length, 'shop items into DB...')
try {
tx(allItems)
console.log('Inserted shop items successfully')
} catch (e) {
console.error('Failed to insert shop items:', e)
} finally {
db.close()
}
}

11
scripts/check-shop-stats.js Normal file
View File

@@ -0,0 +1,11 @@
const shop = require('../database/shop-helpers');
shop.getAllItems().then(items => {
console.log('items:', items.length);
const s = items.find(i => i.id === 786) || items[0];
console.log('sample id', s.id, 'stats type', typeof s.stats);
console.log('stats sample', JSON.stringify(s.stats, null, 2).slice(0, 400));
}).catch(err => {
console.error(err);
process.exit(1);
});

166
scripts/migrate-to-sqlite.js Normal file
View File

@@ -0,0 +1,166 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const Database = require('better-sqlite3');
const dbPath = path.join(__dirname, '../database/sqlite/deathwatch.db');
const backupPath = dbPath + '.backup.' + Date.now();
function safeReadJSON(p) {
if (!fs.existsSync(p)) return null;
try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch (e) { console.error('JSON parse error', p, e.message); return null; }
}
console.log('=== MIGRATE JSON DATA INTO SQLITE ===');
if (!fs.existsSync(dbPath)) {
console.error('DB not found at', dbPath);
process.exit(1);
}
fs.copyFileSync(dbPath, backupPath);
console.log('Backup created:', backupPath);
const db = new Database(dbPath);
// Create consolidated tables
db.exec(`
CREATE TABLE IF NOT EXISTS armour (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT UNIQUE NOT NULL,
req INTEGER DEFAULT 0,
renown TEXT DEFAULT 'None',
category TEXT,
stats TEXT,
source TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS weapons (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT UNIQUE NOT NULL,
req INTEGER DEFAULT 0,
renown TEXT DEFAULT 'None',
category TEXT,
stats TEXT,
source TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS bestiary (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
book TEXT,
page TEXT,
pdf TEXT,
stats TEXT,
profile TEXT,
snippet TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rule_id TEXT UNIQUE,
title TEXT,
content TEXT,
page INTEGER,
source TEXT,
source_abbr TEXT,
category TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
`);
const insertArmour = db.prepare(`INSERT OR IGNORE INTO armour (name, req, renown, category, stats, source) VALUES (?, ?, ?, ?, ?, ?)`);
const insertWeapon = db.prepare(`INSERT OR IGNORE INTO weapons (name, req, renown, category, stats, source) VALUES (?, ?, ?, ?, ?, ?)`);
const insertBestiary = db.prepare(`INSERT INTO bestiary (name, book, page, pdf, stats, profile, snippet) VALUES (?, ?, ?, ?, ?, ?, ?)`);
const insertRule = db.prepare(`INSERT OR IGNORE INTO rules (rule_id, title, content, page, source, source_abbr, category) VALUES (?, ?, ?, ?, ?, ?, ?)`);
let totals = { armour:0, weapons:0, bestiary:0, rules:0 };
// Import armour
const armourFile = path.join(__dirname, '../database/public/deathwatch-armor.json');
const armourData = safeReadJSON(armourFile);
if (armourData) {
const categories = Object.keys(armourData);
categories.forEach(cat => {
const arr = armourData[cat];
if (!Array.isArray(arr)) return;
const insert = insertArmour;
db.transaction(() => {
for (const item of arr) {
const stats = JSON.stringify(item.stats || {});
const src = (item.stats && item.stats.source) || '';
insert.run(item.name || '(unnamed)', item.req || 0, item.renown || 'None', item.category || cat, stats, src);
totals.armour++;
}
})();
});
}
// Import weapons
const weaponsFile = path.join(__dirname, '../database/public/deathwatch-weapons-comprehensive.json');
const weaponsData = safeReadJSON(weaponsFile);
if (weaponsData) {
// many weapon files use keys like rangedWeapons, meleeWeapons
Object.keys(weaponsData).forEach(k => {
const arr = weaponsData[k];
if (!Array.isArray(arr)) return;
db.transaction(() => {
for (const w of arr) {
const stats = JSON.stringify(w.stats || {});
const src = (w.stats && w.stats.source) || '';
insertWeapon.run(w.name || '(unnamed)', w.req || 0, w.renown || 'Any', w.category || k, stats, src);
totals.weapons++;
}
})();
});
}
// Import bestiary
const bestiaryFile = path.join(__dirname, '../database/deathwatch-bestiary-extracted.json');
const bestiaryData = safeReadJSON(bestiaryFile);
if (bestiaryData && Array.isArray(bestiaryData.results)) {
db.transaction(() => {
for (const e of bestiaryData.results) {
const stats = JSON.stringify(e.stats || {});
const profile = JSON.stringify(e.profile || {});
const name = e.bestiaryName || e.name || '(unnamed)';
insertBestiary.run(name, e.book || '', e.page || '', e.pdf || '', stats, profile, e.stats && e.stats.snippet ? e.stats.snippet : '');
totals.bestiary++;
}
})();
}
// Import rules
const rulesFile = path.join(__dirname, '../database/rules/rules-database.json');
const rulesData = safeReadJSON(rulesFile);
if (rulesData && Array.isArray(rulesData.rules)) {
db.transaction(() => {
for (const r of rulesData.rules) {
insertRule.run(r.id || null, r.title || '', r.content || '', r.page || null, r.source || '', r.sourceAbbr || '', r.category || 'general');
totals.rules++;
}
})();
}
console.log('Import totals:', totals);
// Show row counts from DB for verification
const counts = {
armour: db.prepare('SELECT COUNT(*) as c FROM armour').get().c,
weapons: db.prepare('SELECT COUNT(*) as c FROM weapons').get().c,
bestiary: db.prepare('SELECT COUNT(*) as c FROM bestiary').get().c,
rules: db.prepare('SELECT COUNT(*) as c FROM rules').get().c
};
console.log('DB row counts:', counts);
// Print a small sample from each table
console.log('\nSample armour:', db.prepare('SELECT name, category, stats FROM armour LIMIT 3').all());
console.log('\nSample weapons:', db.prepare('SELECT name, category, stats FROM weapons LIMIT 3').all());
console.log('\nSample bestiary:', db.prepare('SELECT name, book, snippet FROM bestiary LIMIT 3').all());
console.log('\nSample rules:', db.prepare('SELECT rule_id, title FROM rules LIMIT 3').all());
db.close();
console.log('\nMigration complete. DB backed up at', backupPath);

38
src/components/BestiaryTab.jsx
View File

@@ -51,6 +51,8 @@ export default function BestiaryTab(){
return arr.map(normalizeEntry)
}catch(e){ return [] }
})
// Keep a ref to the initial enemies value so we can check it in mount-only effects
const initialEnemies = React.useRef(enemies)
// Database / retry UI state
const WARNING_DISMISS_KEY = 'dw:warning-dismiss-until:v1'
const [dbDown, setDbDown] = useState(false)
@@ -86,30 +88,11 @@ export default function BestiaryTab(){
console.error('Database API failed:', error)
}
// Fallback to file endpoints
const endpoints = ['/deathwatch-bestiary-extracted.json','/public/deathwatch-bestiary-extracted.json','/build/deathwatch-bestiary-extracted.json']
for(const ep of endpoints){
try{
const res = await fetch(ep + cacheParam)
if(res.ok){
const data = await res.json()
const arr = Array.isArray(data) ? data : (data.results || [])
if(arr.length>0){
setEnemies(arr.map(normalizeEntry))
localStorage.setItem(STORAGE_ENEMIES, JSON.stringify(arr))
setDbDown(false)
setIsRefreshing(false)
console.log(`Loaded ${arr.length} enemies from ${ep}`)
return
}
}
}catch(e){}
}
// If we reach here, all endpoints failed
setDbDown(true)
setIsRefreshing(false)
// Try reading from cache
// If we reach here, the database API failed to return usable data.
// Mark DB as down and fall back to cache/localStorage only.
setDbDown(true)
setIsRefreshing(false)
// Try reading from cache/localStorage
try{
const raw = localStorage.getItem(STORAGE_ENEMIES)
if(raw){
@@ -122,10 +105,10 @@ export default function BestiaryTab(){
}
}
}catch(e){}
// No data available
setEnemies([])
console.log('No enemy data available')
console.log('No enemy data available — DB and cache are empty')
}
async function updateFromDatabase() {
@@ -155,7 +138,8 @@ export default function BestiaryTab(){
useEffect(()=>{
// Only attempt network load on mount if we have no cached enemies.
if(enemies && enemies.length>0) return
// Use the initialEnemies ref to avoid creating a dependency on `enemies`.
if(initialEnemies.current && initialEnemies.current.length>0) return
loadData().catch(()=>{})
return ()=>{ if(retryRef.current){ clearInterval(retryRef.current); retryRef.current = null } }
},[])

77
src/components/DeathwatchRoller.jsx
View File

@@ -365,6 +365,39 @@ function DeathwatchRoller() {
loadEnemiesFromAPI()
}, [])
// Populate module-scoped built weapons from the server-side DB API so
// buildWeaponsList() can use database-sourced weapon entries instead of
// falling back to packaged JSON files at runtime.
useEffect(() => {
(async () => {
try {
const res = await fetch('/api/weapons', { cache: 'no-store' })
if (!res.ok) return
const json = await res.json()
// Some consumers expect the original build DB shape with keys like
// rangedWeapons / meleeWeapons / grenades / other. If the API returns
// a flat array, group by category to mimic that shape.
let built = json
if (Array.isArray(json)) {
const grouped = { rangedWeapons: [], meleeWeapons: [], grenades: [], other: [] }
for (const it of json) {
const cat = String(it.category || '').toLowerCase()
if (cat.includes('ranged') || cat.includes('ranged weapon') || cat.includes('ranged')) grouped.rangedWeapons.push(it)
else if (cat.includes('melee')) grouped.meleeWeapons.push(it)
else if (cat.includes('grenade')) grouped.grenades.push(it)
else grouped.other.push(it)
}
built = grouped
}
// assign to module-scoped variable used by buildWeaponsList()
_builtWeapons = built
console.info('[DW] Loaded built weapons from /api/weapons', Array.isArray(json) ? json.length : Object.keys(json || {}).length)
} catch (err) {
console.info('[DW] Failed to load built weapons from API:', err && err.message)
}
})()
}, [])
const trackerInit = safeGet(STORAGE_TRACKER)
const [maxWounds,setMaxWounds] = useState(() => Math.max(0, trackerInit?.maxWounds ?? 20))
const [curWounds,setCurWounds] = useState(() => Math.max(0, Math.min(trackerInit?.curWounds ?? 20, trackerInit?.maxWounds ?? 20)))
@@ -412,35 +445,35 @@ function DeathwatchRoller() {
(async () => {
const stored = safeGet(STORAGE_WEAPONS)
if ((!stored || !Array.isArray(stored) || stored.length===0)) {
const endpoints = ['/deathwatch-weapons.json','/public/deathwatch-weapons.json','/build/deathwatch-weapons.json']
let fetched = null
for (const ep of endpoints) {
try {
const res = await fetch(ep, { cache: 'no-store' })
if (!res.ok) continue
// Prefer server API for weapons; if unavailable, keep defaultWeapons
try {
const res = await fetch('/api/weapons', { cache: 'no-store' })
if (res.ok) {
const json = await res.json()
const list = []
if (json && (json.rangedWeapons || json.meleeWeapons || json.grenades || json.other)) {
// Attempt mapping via existing helpers
let list = []
if (Array.isArray(json)) list = json.map(item => item.stats ? mapBuildEntryToWeapon(item) : normalizeWeapon(item))
else if (json && (json.rangedWeapons || json.meleeWeapons)) {
if (Array.isArray(json.rangedWeapons)) list.push(...json.rangedWeapons.map(mapBuildEntryToWeapon))
if (Array.isArray(json.meleeWeapons)) list.push(...json.meleeWeapons.map(mapBuildEntryToWeapon))
if (Array.isArray(json.grenades)) list.push(...json.grenades.map(mapBuildEntryToWeapon))
if (Array.isArray(json.other)) list.push(...json.other.map(mapBuildEntryToWeapon))
} else if (Array.isArray(json)) {
list.push(...json.map(item => item.stats ? mapBuildEntryToWeapon(item) : normalizeWeapon(item)))
}
if (list.length > 0) {
fetched = list
break
}
} catch (e) {
console.info('[DW] first-run fetch failed for', ep, e && e.message)
const merged = mergeWeapons(defaultWeapons, list && list.length ? list : [])
setWeapons(merged)
safeSet(STORAGE_WEAPONS, merged)
if (list && list.length) setInfo('Imported weapons from database')
} else {
const merged = mergeWeapons(defaultWeapons, [])
setWeapons(merged)
safeSet(STORAGE_WEAPONS, merged)
}
} catch (e) {
console.info('[DW] first-run weapons fetch failed', e && e.message)
const merged = mergeWeapons(defaultWeapons, [])
setWeapons(merged)
safeSet(STORAGE_WEAPONS, merged)
}
const merged = mergeWeapons(defaultWeapons, fetched && fetched.length ? fetched : [])
setWeapons(merged)
safeSet(STORAGE_WEAPONS, merged)
if (fetched && fetched.length) setInfo('Imported weapons from database')
}
})()
}, [])
@@ -515,7 +548,7 @@ function DeathwatchRoller() {
async function fetchWeaponsFromServer() {
setError(''); setInfo('')
const endpoints = ['/api/weapons', '/deathwatch-weapons.json', '/public/deathwatch-weapons.json']
const endpoints = ['/api/weapons']
let lastErr = null
for (const ep of endpoints) {
try {

77
src/components/RequisitionShop.jsx
View File

@@ -19,19 +19,7 @@ function renownClass(r) {
return 'bg-slate-700'
}
function slugify(s){ return String(s||'').toLowerCase().replace(/[^a-z0-9]+/g,'-').replace(/(^-|-$)/g,'') }
function normalizeItem(it) {
const name = String(it.name||'').trim()
if (!name) return null
const id = String(it.id||slugify(name))
const category = String(it.category||'Gear').trim()
const req = Math.max(0, Number.isFinite(+it.req) ? +it.req : (Number.isFinite(+it.cost)? +it.cost : 0))
const cost = req
const desc = String(it.desc||'').trim()
const renown = normalizeRank(it.renown||it.Renown)
return { id, name, category, cost, req, renown, desc }
}
// ...existing code...
export default function RequisitionShop({ authedPlayer, sessionId }) {
const [items, setItems] = useState([])
@@ -69,52 +57,47 @@ export default function RequisitionShop({ authedPlayer, sessionId }) {
setPlayers([]);
}
}
// Fetch shop items from the new database (public endpoint, no session needed)
console.log('Fetching shop items');
const itemsResponse = await axios.get('/api/shop/items');
console.log('Fetched shop items:', itemsResponse.data);
if (!itemsResponse.data || itemsResponse.data.length === 0) {
console.log('Warning: Shop items response was empty, falling back to JSON');
throw new Error('Empty shop items');
console.log('Warning: Shop items response was empty')
}
const normalizedItems = itemsResponse.data.map(item => ({
id: item.id,
name: item.name,
category: item.category,
cost: item.requisition_cost,
req: item.requisition_cost,
renown: item.renown_requirement,
desc: item.stats ? JSON.parse(item.stats).description || '' : '',
stats: item.stats ? JSON.parse(item.stats) : {}
}));
// helper to safely get stats as object whether API returned string or object
const parseStats = s => {
if (!s) return {};
if (typeof s === 'string') {
try { return JSON.parse(s); } catch (e) { return { raw: s }; }
}
return s;
}
const normalizedItems = itemsResponse.data.map(item => {
const statsObj = parseStats(item.stats);
return {
id: item.id,
name: item.name,
category: item.category,
cost: item.requisition_cost,
req: item.requisition_cost,
renown: item.renown_requirement,
desc: statsObj.description || '',
stats: statsObj
};
});
setItems(normalizedItems);
} catch (error) {
console.error('Error fetching data:', error);
if (error.response?.status === 404 && error.response?.data?.includes('shop')) {
// If shop API fails, fall back to JSON file
try {
const res = await fetch('/deathwatch-armoury.json', { cache: 'no-store' })
if (!res.ok) return;
const arr = await res.json()
const next = Array.isArray(arr) ? arr.map(normalizeItem).filter(Boolean) : []
if (next.length>0) {
setItems(next)
}
} catch (e) {
console.error('Failed to load fallback JSON:', e);
}
}
// If shop API fails, we can't load items. Keep players fallback behavior.
setPlayers([]);
}
}
if (sessionId) {
fetchData();
}
// Always fetch shop items; fetch players only if we have a sessionId
fetchData();
}, [sessionId]);
const currentPlayer = useMemo(() => {