Fetches better item data, adds spec selector
This commit is contained in:
343
scripts/fetch-items-enhanced.js
Normal file
343
scripts/fetch-items-enhanced.js
Normal file
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { fetchFromWowhead, fetchFromWowheadJson, getExistingItemIds } from './fetch-item-data.js';
|
||||
import { fetchFromBlizzard } from './blizzard-api.js';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
/**
|
||||
* Enhanced item fetcher with multiple API sources and validation
|
||||
*/
|
||||
|
||||
/**
|
||||
* Fetches item data from multiple sources with fallbacks
|
||||
* @param {number} itemId - The item ID to fetch
|
||||
* @returns {Promise<Object|null>} Combined item data or null if failed
|
||||
*/
|
||||
async function fetchItemFromAllSources(itemId) {
|
||||
console.log(`Fetching item ${itemId} from multiple sources...`);
|
||||
|
||||
const results = {
|
||||
wowhead: null,
|
||||
blizzard: null,
|
||||
combined: null
|
||||
};
|
||||
|
||||
// Try Wowhead first (more reliable for icons and basic data)
|
||||
try {
|
||||
results.wowhead = await fetchFromWowheadJson(itemId);
|
||||
if (!results.wowhead) {
|
||||
results.wowhead = await fetchFromWowhead(itemId);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Wowhead fetch failed for ${itemId}:`, error.message);
|
||||
}
|
||||
|
||||
// Try Blizzard API (more detailed data but requires auth)
|
||||
try {
|
||||
results.blizzard = await fetchFromBlizzard(itemId);
|
||||
} catch (error) {
|
||||
console.error(`Blizzard fetch failed for ${itemId}:`, error.message);
|
||||
}
|
||||
|
||||
// Combine the best data from both sources
|
||||
if (results.wowhead || results.blizzard) {
|
||||
results.combined = combineItemData(results.wowhead, results.blizzard, itemId);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Combines item data from multiple sources, preferring the best available data
|
||||
* @param {Object|null} wowheadData - Data from Wowhead
|
||||
* @param {Object|null} blizzardData - Data from Blizzard
|
||||
* @param {number} itemId - The item ID
|
||||
* @returns {Object} Combined item data
|
||||
*/
|
||||
function combineItemData(wowheadData, blizzardData, itemId) {
|
||||
const combined = {
|
||||
id: itemId,
|
||||
name: null,
|
||||
slot: null,
|
||||
icon: null,
|
||||
quality: null,
|
||||
sources: []
|
||||
};
|
||||
|
||||
// Prefer Wowhead for name and icon (usually more consistent)
|
||||
if (wowheadData) {
|
||||
combined.name = wowheadData.name;
|
||||
combined.icon = wowheadData.icon;
|
||||
combined.slot = wowheadData.slot;
|
||||
combined.quality = wowheadData.quality;
|
||||
combined.sources.push('wowhead');
|
||||
}
|
||||
|
||||
// Use Blizzard data to fill gaps or override with better data
|
||||
if (blizzardData) {
|
||||
if (!combined.name) combined.name = blizzardData.name;
|
||||
if (!combined.slot) combined.slot = blizzardData.slot;
|
||||
if (!combined.quality) combined.quality = blizzardData.quality;
|
||||
|
||||
// Add Blizzard-specific data
|
||||
combined.blizzardData = blizzardData.blizzardData;
|
||||
combined.sources.push('blizzard');
|
||||
}
|
||||
|
||||
return combined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates item data against existing project data
|
||||
* @param {Object[]} fetchedItems - Array of fetched item data
|
||||
* @param {Object[]} existingItems - Array of existing project items
|
||||
* @returns {Object} Validation report
|
||||
*/
|
||||
function validateItemData(fetchedItems, existingItems) {
|
||||
const report = {
|
||||
matches: [],
|
||||
mismatches: [],
|
||||
missing: [],
|
||||
newItems: []
|
||||
};
|
||||
|
||||
const existingById = {};
|
||||
existingItems.forEach(item => {
|
||||
existingById[item.id] = item;
|
||||
});
|
||||
|
||||
fetchedItems.forEach(fetched => {
|
||||
const existing = existingById[fetched.combined.id];
|
||||
|
||||
if (existing) {
|
||||
const issues = [];
|
||||
|
||||
if (existing.name !== fetched.combined.name) {
|
||||
issues.push(`name: "${existing.name}" vs "${fetched.combined.name}"`);
|
||||
}
|
||||
|
||||
if (existing.slot !== fetched.combined.slot) {
|
||||
issues.push(`slot: "${existing.slot}" vs "${fetched.combined.slot}"`);
|
||||
}
|
||||
|
||||
if (existing.icon !== fetched.combined.icon && fetched.combined.icon) {
|
||||
issues.push(`icon: "${existing.icon || 'none'}" vs "${fetched.combined.icon}"`);
|
||||
}
|
||||
|
||||
if (issues.length > 0) {
|
||||
report.mismatches.push({
|
||||
id: fetched.combined.id,
|
||||
issues,
|
||||
existing,
|
||||
fetched: fetched.combined
|
||||
});
|
||||
} else {
|
||||
report.matches.push(fetched.combined.id);
|
||||
}
|
||||
} else {
|
||||
report.newItems.push(fetched.combined);
|
||||
}
|
||||
});
|
||||
|
||||
// Find missing items (in project but not fetched)
|
||||
const fetchedIds = new Set(fetchedItems.map(item => item.combined.id));
|
||||
existingItems.forEach(item => {
|
||||
if (!fetchedIds.has(item.id)) {
|
||||
report.missing.push(item.id);
|
||||
}
|
||||
});
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates TypeScript code for new items
|
||||
* @param {Object[]} items - Array of item data
|
||||
* @param {string} source - Source name for the items
|
||||
* @returns {string} TypeScript code
|
||||
*/
|
||||
function generateTypeScriptCode(items, source) {
|
||||
const lines = [];
|
||||
lines.push(`import type { ItemId } from "../types";`);
|
||||
lines.push('');
|
||||
lines.push(`export const ${source.charAt(0).toUpperCase() + source.slice(1)} = [`);
|
||||
|
||||
items.forEach((item, index) => {
|
||||
lines.push(' {');
|
||||
lines.push(` id: ${item.id} as ItemId,`);
|
||||
lines.push(` slot: "${item.slot}",`);
|
||||
lines.push(` name: "${item.name}",`);
|
||||
lines.push(` source: "${source}" as const,`);
|
||||
if (item.icon) {
|
||||
lines.push(` icon: "${item.icon}",`);
|
||||
}
|
||||
lines.push(' }' + (index < items.length - 1 ? ',' : ''));
|
||||
});
|
||||
|
||||
lines.push('];');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Main function with enhanced features
|
||||
*/
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length === 0) {
|
||||
console.log('Enhanced WoW Item Data Fetcher');
|
||||
console.log('Usage:');
|
||||
console.log(' npm run fetch-items-enhanced [options] [item1] [item2] ...');
|
||||
console.log('');
|
||||
console.log('Options:');
|
||||
console.log(' --existing Fetch data for all items in drop files');
|
||||
console.log(' --validate Compare fetched data with existing project data');
|
||||
console.log(' --generate <name> Generate TypeScript code for new source');
|
||||
console.log(' --blizzard-only Use only Blizzard API');
|
||||
console.log(' --wowhead-only Use only Wowhead API');
|
||||
console.log('');
|
||||
console.log('Examples:');
|
||||
console.log(' npm run fetch-items-enhanced 242494 242495 242481');
|
||||
console.log(' npm run fetch-items-enhanced --existing --validate');
|
||||
console.log(' npm run fetch-items-enhanced --generate "new-dungeon" 123456 123457');
|
||||
return;
|
||||
}
|
||||
|
||||
let itemIds = [];
|
||||
let options = {
|
||||
existing: false,
|
||||
validate: false,
|
||||
generate: null,
|
||||
blizzardOnly: false,
|
||||
wowheadOnly: false
|
||||
};
|
||||
|
||||
// Parse arguments
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
|
||||
if (arg === '--existing') {
|
||||
options.existing = true;
|
||||
} else if (arg === '--validate') {
|
||||
options.validate = true;
|
||||
} else if (arg === '--generate') {
|
||||
options.generate = args[++i];
|
||||
} else if (arg === '--blizzard-only') {
|
||||
options.blizzardOnly = true;
|
||||
} else if (arg === '--wowhead-only') {
|
||||
options.wowheadOnly = true;
|
||||
} else if (!isNaN(parseInt(arg))) {
|
||||
itemIds.push(parseInt(arg));
|
||||
}
|
||||
}
|
||||
|
||||
// Get item IDs
|
||||
if (options.existing) {
|
||||
console.log('Fetching data for existing items in drop files...');
|
||||
const existingIds = await getExistingItemIds();
|
||||
itemIds = [...new Set([...itemIds, ...existingIds])];
|
||||
console.log(`Found ${existingIds.length} existing items`);
|
||||
}
|
||||
|
||||
if (itemIds.length === 0) {
|
||||
console.log('No valid item IDs provided');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Processing ${itemIds.length} items...`);
|
||||
|
||||
// Fetch items with rate limiting
|
||||
const results = [];
|
||||
for (let i = 0; i < itemIds.length; i++) {
|
||||
const itemId = itemIds[i];
|
||||
console.log(`\nProcessing item ${itemId} (${i + 1}/${itemIds.length})`);
|
||||
|
||||
let result;
|
||||
if (options.blizzardOnly) {
|
||||
const blizzardData = await fetchFromBlizzard(itemId);
|
||||
result = { wowhead: null, blizzard: blizzardData, combined: blizzardData };
|
||||
} else if (options.wowheadOnly) {
|
||||
const wowheadData = await fetchFromWowheadJson(itemId) || await fetchFromWowhead(itemId);
|
||||
result = { wowhead: wowheadData, blizzard: null, combined: wowheadData };
|
||||
} else {
|
||||
result = await fetchItemFromAllSources(itemId);
|
||||
}
|
||||
|
||||
if (result.combined) {
|
||||
results.push(result);
|
||||
console.log(`✓ ${result.combined.name} (${result.combined.slot || 'unknown slot'}) [${result.combined.sources.join(', ')}]`);
|
||||
} else {
|
||||
console.log(`✗ Failed to fetch data for item ${itemId}`);
|
||||
}
|
||||
|
||||
// Rate limiting
|
||||
if (i < itemIds.length - 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
// Save detailed results
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
await fs.writeFile(
|
||||
`fetched-items-detailed-${timestamp}.json`,
|
||||
JSON.stringify(results, null, 2)
|
||||
);
|
||||
|
||||
// Validation
|
||||
if (options.validate && options.existing) {
|
||||
console.log('\nValidating against existing project data...');
|
||||
const existingItems = [];
|
||||
// This would need to parse the existing TypeScript files
|
||||
// For now, just show what we fetched
|
||||
|
||||
const validation = validateItemData(results, existingItems);
|
||||
console.log(`\nValidation Results:`);
|
||||
console.log(` Matches: ${validation.matches.length}`);
|
||||
console.log(` Mismatches: ${validation.mismatches.length}`);
|
||||
console.log(` New items: ${validation.newItems.length}`);
|
||||
|
||||
if (validation.mismatches.length > 0) {
|
||||
console.log('\nMismatches found:');
|
||||
validation.mismatches.forEach(mismatch => {
|
||||
console.log(` Item ${mismatch.id}: ${mismatch.issues.join(', ')}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Generate TypeScript code
|
||||
if (options.generate) {
|
||||
const validItems = results
|
||||
.map(r => r.combined)
|
||||
.filter(item => item.name && item.slot);
|
||||
|
||||
const tsCode = generateTypeScriptCode(validItems, options.generate);
|
||||
const filename = `${options.generate}.ts`;
|
||||
|
||||
await fs.writeFile(filename, tsCode);
|
||||
console.log(`\nGenerated TypeScript code in ${filename}`);
|
||||
}
|
||||
|
||||
// Summary
|
||||
console.log(`\nSummary:`);
|
||||
console.log(`Successfully fetched: ${results.length}/${itemIds.length} items`);
|
||||
|
||||
const slotCounts = {};
|
||||
results.forEach(result => {
|
||||
const slot = result.combined.slot || 'unknown';
|
||||
slotCounts[slot] = (slotCounts[slot] || 0) + 1;
|
||||
});
|
||||
|
||||
console.log('\nSlot distribution:');
|
||||
Object.entries(slotCounts).forEach(([slot, count]) => {
|
||||
console.log(` ${slot}: ${count}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch(console.error);
|
||||
}
|
||||
|
||||
export { fetchItemFromAllSources, combineItemData, validateItemData, generateTypeScriptCode };
|
||||
Reference in New Issue
Block a user