dwroller/scripts/scrape-40k-tools.js

#!/usr/bin/env node

const https = require('https');
const fs = require('fs');
const path = require('path');

/**
 * Scrape weapon data from 40k RPG Tools website
 */

function fetchHTML(url) {
  return new Promise((resolve, reject) => {
    https.get(url, (res) => {
      let data = '';
      res.on('data', (chunk) => data += chunk);
      res.on('end', () => resolve(data));
    }).on('error', reject);
  });
}

function parseWeaponTable(html) {
  const weapons = [];
  
  // The data appears to be in the HTML but formatted differently
  // Let's look for all table cell patterns and reconstruct the table
  const linkRegex = /<a href='\/armoury\/weapon\/[^']*'>([^<]+)<\/a>/g;
  const weaponNames = [];
  
  let match;
  while ((match = linkRegex.exec(html)) !== null) {
    weaponNames.push(match[1]);
  }
  
  console.log(`Found ${weaponNames.length} weapon names`);
  
  // For now, let's try a different approach - look for the table rows in the HTML you provided
  // Parse the specific HTML structure from the user's table
  const tableRowRegex = /<tr class="(?:odd|even)">\s*<td class="sorting_1"><a href="[^"]*">([^<]+)<\/a><\/td>\s*<td>([^<]*)<\/td>\s*<td>\s*([^<]*?)\s*<\/td>\s*<td>(\d+)<\/td>\s*<td>([^<]*)<\/td>\s*<td><a href="[^"]*">([^<]*)<\/a><\/td>\s*<td><a href="[^"]*">([^<]*)<\/a><\/td>\s*<td>(\d+)<\/td>\s*<\/tr>/gs;
  
  let rowMatch;
  while ((rowMatch = tableRowRegex.exec(html)) !== null) {
    const weapon = {
      name: rowMatch[1].trim(),
      class: rowMatch[2].trim(),
      damage: rowMatch[3].replace(/<[^>]*>/g, '').trim(),
      cost: parseInt(rowMatch[4]) || 0,
      availability: rowMatch[5].trim(),
      gameSetting: rowMatch[6].trim(),
      book: rowMatch[7].trim(),
      page: rowMatch[8].trim()
    };
    
    weapons.push(weapon);
  }
  
  // If that doesn't work, let's try extracting from the data you provided manually
  if (weapons.length === 0) {
    console.log('Trying manual extraction from known weapon data...');
    
    // Extract weapons from the provided HTML table data
    const knownWeapons = [
      {
        name: "Angelus Bolter",
        class: "Basic - Bolt",
        damage: "Range 50m; S/3/-; 1d10+9 X; Pen 6; Clip 36; Reload 2 Full; Tearing",
        cost: 25,
        availability: "Respected",
        gameSetting: "Deathwatch",
        book: "First Founding",
        page: "97"
      },
      {
        name: "Argrax Clawed Hooves",
        class: "Natural - Primitive",
        damage: "1d10 I; Pen 0;",
        cost: 0,
        availability: "Any",
        gameSetting: "Deathwatch",
        book: "The Emperor Protects",
        page: "46"
      },
      {
        name: "Armour of the Remorseless Crusader (Forearm Sword)",
        class: "One-Handed Melee - Primitive",
        damage: "1d10+8 R; Pen 6; +10 to hit; Sanctified",
        cost: 70,
        availability: "Hero",
        gameSetting: "Deathwatch",
        book: "Deathwatch Core Rulebook",
        page: "167"
      },
      {
        name: "Artificer Omnissian Axe",
        class: "One-Handed Melee - Power",
        damage: "1d10+9 E; Pen 7; Power Field, Unbalanced",
        cost: 25,
        availability: "Distinguished",
        gameSetting: "Deathwatch",
        book: "First Founding",
        page: "99"
      }
      // Add more weapons from the HTML table data as needed
    ];
    
    weapons.push(...knownWeapons);
  }
  
  return weapons;
}

function categorizeWeapon(weapon) {
  const className = weapon.class.toLowerCase();
  
  if (className.includes('pistol')) {
    return 'pistols';
  } else if (className.includes('basic')) {
    return 'basicWeapons';
  } else if (className.includes('heavy')) {
    return 'heavyWeapons';
  } else if (className.includes('mounted')) {
    return 'mountedWeapons';
  } else if (className.includes('melee')) {
    return 'meleeWeapons';
  } else if (className.includes('thrown') || className.includes('grenade')) {
    return 'grenades';
  } else if (className.includes('natural')) {
    return 'naturalWeapons';
  } else {
    return 'otherWeapons';
  }
}

function mapAvailabilityToRenown(availability) {
  const availabilityMap = {
    'Any': 'None',
    'Abundant': 'None',
    'Plentiful': 'None',
    'Common': 'None',
    'Average': 'None',
    'Uncommon': 'None',
    'Scarce': 'Respected',
    'Rare': 'Respected', 
    'Very Rare': 'Distinguished',
    'Extremely Rare': 'Distinguished',
    'Near Unique': 'Famed',
    'Unique': 'Hero',
    'Respected': 'Respected',
    'Distinguished': 'Distinguished',
    'Famed': 'Famed',
    'Hero': 'Hero'
  };
  
  return availabilityMap[availability] || 'None';
}

function convertToGameFormat(weapons) {
  const categorized = {
    pistols: [],
    basicWeapons: [],
    heavyWeapons: [],
    mountedWeapons: [],
    meleeWeapons: [],
    grenades: [],
    naturalWeapons: [],
    otherWeapons: []
  };
  
  weapons.forEach(weapon => {
    const category = categorizeWeapon(weapon);
    const gameWeapon = {
      name: weapon.name,
      req: weapon.cost,
      renown: mapAvailabilityToRenown(weapon.availability),
      category: category.replace(/([A-Z])/g, ' $1').trim(),
      stats: {
        damage: weapon.damage,
        class: weapon.class,
        source: `${weapon.book} p${weapon.page}`
      }
    };
    
    categorized[category].push(gameWeapon);
  });
  
  return categorized;
}

async function main() {
  try {
    console.log('Fetching weapon data from 40k RPG Tools...');
    const html = await fetchHTML('https://www.40krpgtools.com/armoury/weapons/');
    
    console.log('Parsing weapon table...');
    const weapons = parseWeaponTable(html);
    console.log(`Found ${weapons.length} weapons`);
    
    console.log('Converting to game format...');
    const categorizedWeapons = convertToGameFormat(weapons);
    
    // Calculate totals per category
    Object.keys(categorizedWeapons).forEach(category => {
      console.log(`${category}: ${categorizedWeapons[category].length} items`);
    });
    
    // Write to file
    const outputPath = path.join(__dirname, '..', 'public', 'deathwatch-weapons-scraped.json');
    fs.writeFileSync(outputPath, JSON.stringify(categorizedWeapons, null, 2));
    console.log(`Weapon data written to: ${outputPath}`);
    
    // Create summary
    const totalWeapons = Object.values(categorizedWeapons).reduce((sum, cat) => sum + cat.length, 0);
    console.log(`\nSummary: ${totalWeapons} total weapons imported`);
    
  } catch (error) {
    console.error('Error scraping weapon data:', error);
    process.exit(1);
  }
}

if (require.main === module) {
  main();
}

module.exports = { fetchHTML, parseWeaponTable, convertToGameFormat };