Fetches better item data, adds spec selector
This commit is contained in:
117
scripts/README.md
Normal file
117
scripts/README.md
Normal file
@@ -0,0 +1,117 @@
|
||||
# WoW Item Data Fetching Scripts
|
||||
|
||||
This directory contains scripts to fetch World of Warcraft item data from various APIs.
|
||||
|
||||
## Scripts
|
||||
|
||||
### `fetch-item-data.js` - Basic Item Fetcher
|
||||
|
||||
A simple script that fetches item information from Wowhead API.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
# Fetch specific items by ID
|
||||
npm run fetch-items 242494 242495 242481
|
||||
|
||||
# Fetch data for all existing items in drop files
|
||||
npm run fetch-items --existing
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Fetches item name, slot, icon from Wowhead
|
||||
- Supports both XML and JSON Wowhead APIs
|
||||
- Rate limiting to avoid overwhelming servers
|
||||
- Saves results to JSON file
|
||||
|
||||
### `fetch-items-enhanced.js` - Advanced Item Fetcher
|
||||
|
||||
Enhanced script with multiple API sources and additional features.
|
||||
|
||||
**Usage:**
|
||||
```bash
|
||||
# Basic usage
|
||||
npm run fetch-items-enhanced 242494 242495 242481
|
||||
|
||||
# Fetch and validate existing items
|
||||
npm run fetch-items-enhanced --existing --validate
|
||||
|
||||
# Generate TypeScript code for new dungeon
|
||||
npm run fetch-items-enhanced --generate "new-dungeon" 123456 123457
|
||||
|
||||
# Use only specific API
|
||||
npm run fetch-items-enhanced --wowhead-only 242494
|
||||
npm run fetch-items-enhanced --blizzard-only 242494
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Multiple API sources (Wowhead + Blizzard)
|
||||
- Data validation against existing project data
|
||||
- TypeScript code generation
|
||||
- Detailed reporting and analysis
|
||||
- Fallback mechanisms
|
||||
|
||||
### `blizzard-api.js` - Blizzard API Client
|
||||
|
||||
Handles authentication and data fetching from Blizzard's official API.
|
||||
|
||||
**Setup for Blizzard API:**
|
||||
1. Create an application at https://develop.battle.net/
|
||||
2. Get your Client ID and Client Secret
|
||||
3. Set environment variables:
|
||||
```bash
|
||||
export BLIZZARD_CLIENT_ID="your_client_id"
|
||||
export BLIZZARD_CLIENT_SECRET="your_client_secret"
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- OAuth2 token management
|
||||
- Item data and media fetching
|
||||
- Detailed item metadata
|
||||
|
||||
## Data Sources
|
||||
|
||||
### Wowhead API
|
||||
- **Pros:** Reliable, no authentication required, good icon data
|
||||
- **Cons:** Unofficial API, rate limiting needed
|
||||
- **Best for:** Basic item info, icons, quick lookups
|
||||
|
||||
### Blizzard API
|
||||
- **Pros:** Official API, detailed metadata, reliable
|
||||
- **Cons:** Requires authentication setup, more complex
|
||||
- **Best for:** Detailed item data, official information
|
||||
|
||||
## Output Files
|
||||
|
||||
- `fetched-items.json` - Basic fetch results
|
||||
- `fetched-items-detailed-{timestamp}.json` - Enhanced fetch results with all API responses
|
||||
- `{source-name}.ts` - Generated TypeScript code for new item sources
|
||||
|
||||
## Examples
|
||||
|
||||
### Fetch a few specific items
|
||||
```bash
|
||||
npm run fetch-items 242494 242495 242481
|
||||
```
|
||||
|
||||
### Validate all existing project items
|
||||
```bash
|
||||
npm run fetch-items-enhanced --existing --validate
|
||||
```
|
||||
|
||||
### Create a new dungeon data file
|
||||
```bash
|
||||
npm run fetch-items-enhanced --generate "mists-of-tirna-scithe" 178692 178693 178694
|
||||
```
|
||||
|
||||
### Use only Wowhead (no Blizzard setup needed)
|
||||
```bash
|
||||
npm run fetch-items-enhanced --wowhead-only --existing
|
||||
```
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
Both scripts include rate limiting (1 second between requests) to be respectful to the APIs. For large batches of items, expect the process to take some time.
|
||||
|
||||
## Error Handling
|
||||
|
||||
The scripts will continue processing even if individual items fail to fetch. Check the console output and generated files for detailed results and any errors encountered.
|
||||
241
scripts/blizzard-api.js
Normal file
241
scripts/blizzard-api.js
Normal file
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import https from 'https';
|
||||
|
||||
/**
|
||||
* Blizzard API client for fetching item data
|
||||
* To use this, you'll need to:
|
||||
* 1. Create an app at https://develop.battle.net/
|
||||
* 2. Get your client ID and client secret
|
||||
* 3. Set environment variables: BLIZZARD_CLIENT_ID and BLIZZARD_CLIENT_SECRET
|
||||
*/
|
||||
|
||||
let accessToken = null;
|
||||
let tokenExpiry = null;
|
||||
|
||||
/**
|
||||
* Gets an access token from Blizzard API
|
||||
* @returns {Promise<string|null>} Access token or null if failed
|
||||
*/
|
||||
async function getAccessToken() {
|
||||
const clientId = process.env.BLIZZARD_CLIENT_ID;
|
||||
const clientSecret = process.env.BLIZZARD_CLIENT_SECRET;
|
||||
|
||||
if (!clientId || !clientSecret) {
|
||||
console.error('Blizzard API credentials not found. Please set BLIZZARD_CLIENT_ID and BLIZZARD_CLIENT_SECRET environment variables.');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if we have a valid token
|
||||
if (accessToken && tokenExpiry && Date.now() < tokenExpiry) {
|
||||
return accessToken;
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const auth = Buffer.from(`${clientId}:${clientSecret}`).toString('base64');
|
||||
const postData = 'grant_type=client_credentials';
|
||||
|
||||
const options = {
|
||||
hostname: 'oauth.battle.net',
|
||||
port: 443,
|
||||
path: '/token',
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Basic ${auth}`,
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Content-Length': Buffer.byteLength(postData)
|
||||
}
|
||||
};
|
||||
|
||||
const req = https.request(options, (res) => {
|
||||
let data = '';
|
||||
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on('end', () => {
|
||||
try {
|
||||
const response = JSON.parse(data);
|
||||
if (response.access_token) {
|
||||
accessToken = response.access_token;
|
||||
tokenExpiry = Date.now() + (response.expires_in * 1000);
|
||||
resolve(accessToken);
|
||||
} else {
|
||||
console.error('Failed to get access token:', response);
|
||||
resolve(null);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error parsing token response:', error.message);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', (error) => {
|
||||
console.error('Error getting access token:', error.message);
|
||||
resolve(null);
|
||||
});
|
||||
|
||||
req.write(postData);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches item data from Blizzard API
|
||||
* @param {number} itemId - The item ID to fetch
|
||||
* @param {string} region - The region (default: 'us')
|
||||
* @returns {Promise<Object|null>} Item data or null if failed
|
||||
*/
|
||||
async function fetchFromBlizzard(itemId, region = 'us') {
|
||||
const token = await getAccessToken();
|
||||
if (!token) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const options = {
|
||||
hostname: `${region}.api.blizzard.com`,
|
||||
port: 443,
|
||||
path: `/data/wow/item/${itemId}?namespace=static-${region}&locale=en_US&access_token=${token}`,
|
||||
method: 'GET'
|
||||
};
|
||||
|
||||
https.get(options, (res) => {
|
||||
let data = '';
|
||||
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on('end', () => {
|
||||
try {
|
||||
const response = JSON.parse(data);
|
||||
|
||||
if (response.name) {
|
||||
const itemData = {
|
||||
id: itemId,
|
||||
name: response.name,
|
||||
slot: mapSlotFromBlizzard(response.inventory_type?.type),
|
||||
quality: response.quality?.name?.toLowerCase(),
|
||||
icon: response.media ? `blizzard-${response.media.id}` : null,
|
||||
source: 'blizzard',
|
||||
blizzardData: {
|
||||
itemClass: response.item_class?.name,
|
||||
itemSubclass: response.item_subclass?.name,
|
||||
inventoryType: response.inventory_type?.name,
|
||||
bindType: response.bind_type?.name,
|
||||
level: response.level,
|
||||
requiredLevel: response.required_level
|
||||
}
|
||||
};
|
||||
resolve(itemData);
|
||||
} else {
|
||||
console.log(`No data found for item ${itemId} on Blizzard API`);
|
||||
resolve(null);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error parsing Blizzard data for item ${itemId}:`, error.message);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
}).on('error', (error) => {
|
||||
console.error(`Error fetching from Blizzard for item ${itemId}:`, error.message);
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps Blizzard inventory types to our slot names
|
||||
* @param {string} inventoryType - Blizzard inventory type
|
||||
* @returns {string|null} Our slot name or null
|
||||
*/
|
||||
function mapSlotFromBlizzard(inventoryType) {
|
||||
const typeMap = {
|
||||
'HEAD': 'head',
|
||||
'NECK': 'neck',
|
||||
'SHOULDER': 'shoulders',
|
||||
'BODY': 'chest',
|
||||
'CHEST': 'chest',
|
||||
'WAIST': 'waist',
|
||||
'LEGS': 'legs',
|
||||
'FEET': 'feet',
|
||||
'WRIST': 'wrist',
|
||||
'HAND': 'hands',
|
||||
'FINGER': 'finger',
|
||||
'TRINKET': 'trinket',
|
||||
'WEAPON': '1h-weapon',
|
||||
'SHIELD': 'off-hand',
|
||||
'RANGED': '2h-weapon',
|
||||
'CLOAK': 'back',
|
||||
'WEAPON_2H': '2h-weapon',
|
||||
'WEAPON_MAIN_HAND': '1h-weapon',
|
||||
'WEAPON_OFF_HAND': 'off-hand',
|
||||
'HOLDABLE': 'off-hand',
|
||||
'AMMO': null,
|
||||
'THROWN': '1h-weapon',
|
||||
'RANGED_RIGHT': '2h-weapon',
|
||||
'QUIVER': null,
|
||||
'RELIC': 'trinket'
|
||||
};
|
||||
|
||||
return typeMap[inventoryType] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches item media (icon) from Blizzard API
|
||||
* @param {number} itemId - The item ID
|
||||
* @param {string} region - The region (default: 'us')
|
||||
* @returns {Promise<string|null>} Icon URL or null if failed
|
||||
*/
|
||||
async function fetchItemMedia(itemId, region = 'us') {
|
||||
const token = await getAccessToken();
|
||||
if (!token) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const options = {
|
||||
hostname: `${region}.api.blizzard.com`,
|
||||
port: 443,
|
||||
path: `/data/wow/media/item/${itemId}?namespace=static-${region}&locale=en_US&access_token=${token}`,
|
||||
method: 'GET'
|
||||
};
|
||||
|
||||
https.get(options, (res) => {
|
||||
let data = '';
|
||||
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on('end', () => {
|
||||
try {
|
||||
const response = JSON.parse(data);
|
||||
|
||||
if (response.assets && response.assets.length > 0) {
|
||||
const iconAsset = response.assets.find(asset => asset.key === 'icon');
|
||||
resolve(iconAsset ? iconAsset.value : null);
|
||||
} else {
|
||||
resolve(null);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error parsing media data for item ${itemId}:`, error.message);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
}).on('error', (error) => {
|
||||
console.error(`Error fetching media for item ${itemId}:`, error.message);
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export {
|
||||
fetchFromBlizzard,
|
||||
fetchItemMedia,
|
||||
getAccessToken,
|
||||
mapSlotFromBlizzard
|
||||
};
|
||||
313
scripts/fetch-item-data.js
Normal file
313
scripts/fetch-item-data.js
Normal file
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import https from "https";
|
||||
import fs from "fs/promises";
|
||||
import path from "path";
|
||||
|
||||
/**
|
||||
* Fetches item data from Wowhead API
|
||||
* @param {number} itemId - The item ID to fetch
|
||||
* @returns {Promise<Object|null>} Item data or null if failed
|
||||
*/
|
||||
async function fetchFromWowhead(itemId, source) {
|
||||
return new Promise((resolve) => {
|
||||
const url = `https://www.wowhead.com/item=${itemId}&xml`;
|
||||
|
||||
https
|
||||
.get(url, (res) => {
|
||||
let data = "";
|
||||
|
||||
res.on("data", (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on("end", () => {
|
||||
try {
|
||||
// Weirdly, the JSON in the XML has more data than the XML itself or the JSON API.
|
||||
const jsonString = data.match(/<json><!\[CDATA\[(.*)\]\]><\/json>/);
|
||||
const jsonData = JSON.parse(`{${jsonString[1]}}`);
|
||||
|
||||
// Parse the XML response from Wowhead
|
||||
// const nameMatch = data.match(/<name><!\[CDATA\[(.*?)\]\]><\/name>/);
|
||||
const iconMatch = data.match(
|
||||
/<icon displayId="(\d+)">(.*?)<\/icon>/,
|
||||
);
|
||||
const slotMatch = data.match(
|
||||
/<inventorySlot id="(\d+)">(.*?)<\/inventorySlot>/,
|
||||
);
|
||||
|
||||
const name = jsonData.displayName;
|
||||
|
||||
const itemData = {
|
||||
id: itemId,
|
||||
name,
|
||||
icon: iconMatch ? iconMatch[2] : null,
|
||||
slot: mapSlotFromWowhead(jsonData.slot),
|
||||
specs: jsonData.specs || [],
|
||||
source,
|
||||
};
|
||||
resolve(itemData);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Error parsing Wowhead data for item ${itemId}:`,
|
||||
error.message,
|
||||
);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
})
|
||||
.on("error", (error) => {
|
||||
console.error(
|
||||
`Error fetching from Wowhead for item ${itemId}:`,
|
||||
error.message,
|
||||
);
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches item data from Wowhead's JSON API (alternative method)
|
||||
* @param {number} itemId - The item ID to fetch
|
||||
* @returns {Promise<Object|null>} Item data or null if failed
|
||||
*/
|
||||
async function fetchFromWowheadJson(itemId) {
|
||||
return new Promise((resolve) => {
|
||||
const url = `https://nether.wowhead.com/tooltip/item/${itemId}`;
|
||||
|
||||
https
|
||||
.get(url, (res) => {
|
||||
let data = "";
|
||||
|
||||
res.on("data", (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on("end", () => {
|
||||
try {
|
||||
const jsonData = JSON.parse(data);
|
||||
|
||||
if (jsonData && jsonData.name) {
|
||||
const itemData = {
|
||||
id: itemId,
|
||||
name: jsonData.name,
|
||||
icon: jsonData.icon || null,
|
||||
slot: mapSlotFromWowhead(jsonData.slot || 0),
|
||||
};
|
||||
resolve(itemData);
|
||||
} else {
|
||||
resolve(null);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Error parsing Wowhead JSON data for item ${itemId}:`,
|
||||
error.message,
|
||||
);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
})
|
||||
.on("error", (error) => {
|
||||
console.error(
|
||||
`Error fetching from Wowhead JSON for item ${itemId}:`,
|
||||
error.message,
|
||||
);
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps Wowhead slot numbers to our slot names
|
||||
* @param {number} slotId - Wowhead slot ID
|
||||
* @returns {string|null} Our slot name or null
|
||||
*/
|
||||
function mapSlotFromWowhead(slotId) {
|
||||
const slotMap = {
|
||||
1: "head",
|
||||
2: "neck",
|
||||
3: "shoulders",
|
||||
5: "chest",
|
||||
6: "waist",
|
||||
7: "legs",
|
||||
8: "feet",
|
||||
9: "wrist",
|
||||
10: "hands",
|
||||
11: "finger",
|
||||
12: "trinket",
|
||||
13: "1h-weapon",
|
||||
14: "off-hand",
|
||||
15: "back",
|
||||
16: "1h-weapon", // Main hand
|
||||
17: "2h-weapon",
|
||||
21: "1h-weapon", // One-handed
|
||||
22: "off-hand",
|
||||
23: "off-hand", // Held in off-hand
|
||||
25: "1h-weapon", // Thrown
|
||||
};
|
||||
|
||||
return slotMap[slotId] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches multiple items with rate limiting
|
||||
* @param {number[]} itemIds - Array of item IDs to fetch
|
||||
* @param {number} delay - Delay between requests in ms
|
||||
* @returns {Promise<Object[]>} Array of item data
|
||||
*/
|
||||
async function fetchMultipleItems(source, itemIds, delay = 1000) {
|
||||
const results = [];
|
||||
|
||||
console.log(`Fetching data for ${itemIds.length} items...`);
|
||||
|
||||
for (let i = 0; i < itemIds.length; i++) {
|
||||
const itemId = itemIds[i];
|
||||
console.log(`Fetching item ${itemId} (${i + 1}/${itemIds.length})`);
|
||||
|
||||
// Try Wowhead JSON API first, fallback to XML
|
||||
let itemData = await fetchFromWowhead(itemId, source);
|
||||
if (!itemData) {
|
||||
itemData = await fetchFromWowheadJson(itemId, source);
|
||||
}
|
||||
|
||||
if (itemData) {
|
||||
results.push(itemData);
|
||||
console.log(
|
||||
`✓ Found: ${itemData.name} (${itemData.slot || "unknown slot"})`,
|
||||
);
|
||||
} else {
|
||||
console.log(`✗ Failed to fetch data for item ${itemId}`);
|
||||
}
|
||||
|
||||
// Rate limiting
|
||||
if (i < itemIds.length - 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads item IDs from existing drop files
|
||||
* @returns {Promise<number[]>} Array of item IDs
|
||||
*/
|
||||
async function getExistingItemIds() {
|
||||
const dropsDir = path.join(process.cwd(), "src", "lib", "drops");
|
||||
const itemIds = [];
|
||||
|
||||
try {
|
||||
const files = await fs.readdir(dropsDir);
|
||||
|
||||
for (const file of files) {
|
||||
if (file.endsWith(".ts") && file !== "index.ts") {
|
||||
const filePath = path.join(dropsDir, file);
|
||||
const content = await fs.readFile(filePath, "utf-8");
|
||||
|
||||
// Extract item IDs using regex
|
||||
const idMatches = content.match(/id: (\d+) as ItemId/g);
|
||||
if (idMatches) {
|
||||
for (const match of idMatches) {
|
||||
const id = parseInt(match.match(/id: (\d+)/)[1]);
|
||||
itemIds.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error reading existing item IDs:", error.message);
|
||||
}
|
||||
|
||||
return [...new Set(itemIds)]; // Remove duplicates
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves fetched data to a JSON file
|
||||
* @param {Object[]} items - Array of item data
|
||||
* @param {string} filename - Output filename
|
||||
*/
|
||||
async function saveToFile(items, filename = "fetched-items.json") {
|
||||
try {
|
||||
const outputPath = path.join(process.cwd(), filename);
|
||||
await fs.writeFile(outputPath, JSON.stringify(items, null, 2));
|
||||
console.log(`\nSaved ${items.length} items to ${filename}`);
|
||||
} catch (error) {
|
||||
console.error("Error saving file:", error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main function
|
||||
*/
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length === 0) {
|
||||
console.log(
|
||||
"Usage: npm run fetch-items [item1] [item2] ... or npm run fetch-items --existing",
|
||||
);
|
||||
console.log("Examples:");
|
||||
console.log(" npm run fetch-items 242494 242495 242481");
|
||||
console.log(
|
||||
" npm run fetch-items --existing # Fetch data for all items in drop files",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let itemIds = [];
|
||||
let source = null;
|
||||
|
||||
if (args[0] === "--existing") {
|
||||
console.log("Fetching data for existing items in drop files...");
|
||||
itemIds = await getExistingItemIds();
|
||||
console.log(`Found ${itemIds.length} existing items`);
|
||||
} else {
|
||||
source = args[0];
|
||||
itemIds = args
|
||||
.slice(1)
|
||||
.map((arg) => parseInt(arg))
|
||||
.filter((id) => !isNaN(id));
|
||||
}
|
||||
|
||||
if (itemIds.length === 0) {
|
||||
console.log("No valid item IDs provided");
|
||||
return;
|
||||
}
|
||||
|
||||
const items = await fetchMultipleItems(source, itemIds);
|
||||
|
||||
if (items.length > 0) {
|
||||
await saveToFile(items);
|
||||
|
||||
console.log("\nSummary:");
|
||||
console.log(
|
||||
`Successfully fetched: ${items.length}/${itemIds.length} items`,
|
||||
);
|
||||
|
||||
// Show slot distribution
|
||||
const slotCounts = {};
|
||||
items.forEach((item) => {
|
||||
const slot = item.slot || "unknown";
|
||||
slotCounts[slot] = (slotCounts[slot] || 0) + 1;
|
||||
});
|
||||
|
||||
console.log("\nSlot distribution:");
|
||||
Object.entries(slotCounts).forEach(([slot, count]) => {
|
||||
console.log(` ${slot}: ${count}`);
|
||||
});
|
||||
} else {
|
||||
console.log("No items were successfully fetched");
|
||||
}
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch(console.error);
|
||||
}
|
||||
|
||||
export {
|
||||
fetchFromWowhead,
|
||||
fetchFromWowheadJson,
|
||||
fetchMultipleItems,
|
||||
getExistingItemIds,
|
||||
};
|
||||
343
scripts/fetch-items-enhanced.js
Normal file
343
scripts/fetch-items-enhanced.js
Normal file
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { fetchFromWowhead, fetchFromWowheadJson, getExistingItemIds } from './fetch-item-data.js';
|
||||
import { fetchFromBlizzard } from './blizzard-api.js';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
|
||||
/**
|
||||
* Enhanced item fetcher with multiple API sources and validation
|
||||
*/
|
||||
|
||||
/**
|
||||
* Fetches item data from multiple sources with fallbacks
|
||||
* @param {number} itemId - The item ID to fetch
|
||||
* @returns {Promise<Object|null>} Combined item data or null if failed
|
||||
*/
|
||||
async function fetchItemFromAllSources(itemId) {
|
||||
console.log(`Fetching item ${itemId} from multiple sources...`);
|
||||
|
||||
const results = {
|
||||
wowhead: null,
|
||||
blizzard: null,
|
||||
combined: null
|
||||
};
|
||||
|
||||
// Try Wowhead first (more reliable for icons and basic data)
|
||||
try {
|
||||
results.wowhead = await fetchFromWowheadJson(itemId);
|
||||
if (!results.wowhead) {
|
||||
results.wowhead = await fetchFromWowhead(itemId);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Wowhead fetch failed for ${itemId}:`, error.message);
|
||||
}
|
||||
|
||||
// Try Blizzard API (more detailed data but requires auth)
|
||||
try {
|
||||
results.blizzard = await fetchFromBlizzard(itemId);
|
||||
} catch (error) {
|
||||
console.error(`Blizzard fetch failed for ${itemId}:`, error.message);
|
||||
}
|
||||
|
||||
// Combine the best data from both sources
|
||||
if (results.wowhead || results.blizzard) {
|
||||
results.combined = combineItemData(results.wowhead, results.blizzard, itemId);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Combines item data from multiple sources, preferring the best available data
|
||||
* @param {Object|null} wowheadData - Data from Wowhead
|
||||
* @param {Object|null} blizzardData - Data from Blizzard
|
||||
* @param {number} itemId - The item ID
|
||||
* @returns {Object} Combined item data
|
||||
*/
|
||||
function combineItemData(wowheadData, blizzardData, itemId) {
|
||||
const combined = {
|
||||
id: itemId,
|
||||
name: null,
|
||||
slot: null,
|
||||
icon: null,
|
||||
quality: null,
|
||||
sources: []
|
||||
};
|
||||
|
||||
// Prefer Wowhead for name and icon (usually more consistent)
|
||||
if (wowheadData) {
|
||||
combined.name = wowheadData.name;
|
||||
combined.icon = wowheadData.icon;
|
||||
combined.slot = wowheadData.slot;
|
||||
combined.quality = wowheadData.quality;
|
||||
combined.sources.push('wowhead');
|
||||
}
|
||||
|
||||
// Use Blizzard data to fill gaps or override with better data
|
||||
if (blizzardData) {
|
||||
if (!combined.name) combined.name = blizzardData.name;
|
||||
if (!combined.slot) combined.slot = blizzardData.slot;
|
||||
if (!combined.quality) combined.quality = blizzardData.quality;
|
||||
|
||||
// Add Blizzard-specific data
|
||||
combined.blizzardData = blizzardData.blizzardData;
|
||||
combined.sources.push('blizzard');
|
||||
}
|
||||
|
||||
return combined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates item data against existing project data
|
||||
* @param {Object[]} fetchedItems - Array of fetched item data
|
||||
* @param {Object[]} existingItems - Array of existing project items
|
||||
* @returns {Object} Validation report
|
||||
*/
|
||||
function validateItemData(fetchedItems, existingItems) {
|
||||
const report = {
|
||||
matches: [],
|
||||
mismatches: [],
|
||||
missing: [],
|
||||
newItems: []
|
||||
};
|
||||
|
||||
const existingById = {};
|
||||
existingItems.forEach(item => {
|
||||
existingById[item.id] = item;
|
||||
});
|
||||
|
||||
fetchedItems.forEach(fetched => {
|
||||
const existing = existingById[fetched.combined.id];
|
||||
|
||||
if (existing) {
|
||||
const issues = [];
|
||||
|
||||
if (existing.name !== fetched.combined.name) {
|
||||
issues.push(`name: "${existing.name}" vs "${fetched.combined.name}"`);
|
||||
}
|
||||
|
||||
if (existing.slot !== fetched.combined.slot) {
|
||||
issues.push(`slot: "${existing.slot}" vs "${fetched.combined.slot}"`);
|
||||
}
|
||||
|
||||
if (existing.icon !== fetched.combined.icon && fetched.combined.icon) {
|
||||
issues.push(`icon: "${existing.icon || 'none'}" vs "${fetched.combined.icon}"`);
|
||||
}
|
||||
|
||||
if (issues.length > 0) {
|
||||
report.mismatches.push({
|
||||
id: fetched.combined.id,
|
||||
issues,
|
||||
existing,
|
||||
fetched: fetched.combined
|
||||
});
|
||||
} else {
|
||||
report.matches.push(fetched.combined.id);
|
||||
}
|
||||
} else {
|
||||
report.newItems.push(fetched.combined);
|
||||
}
|
||||
});
|
||||
|
||||
// Find missing items (in project but not fetched)
|
||||
const fetchedIds = new Set(fetchedItems.map(item => item.combined.id));
|
||||
existingItems.forEach(item => {
|
||||
if (!fetchedIds.has(item.id)) {
|
||||
report.missing.push(item.id);
|
||||
}
|
||||
});
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates TypeScript code for new items
|
||||
* @param {Object[]} items - Array of item data
|
||||
* @param {string} source - Source name for the items
|
||||
* @returns {string} TypeScript code
|
||||
*/
|
||||
function generateTypeScriptCode(items, source) {
|
||||
const lines = [];
|
||||
lines.push(`import type { ItemId } from "../types";`);
|
||||
lines.push('');
|
||||
lines.push(`export const ${source.charAt(0).toUpperCase() + source.slice(1)} = [`);
|
||||
|
||||
items.forEach((item, index) => {
|
||||
lines.push(' {');
|
||||
lines.push(` id: ${item.id} as ItemId,`);
|
||||
lines.push(` slot: "${item.slot}",`);
|
||||
lines.push(` name: "${item.name}",`);
|
||||
lines.push(` source: "${source}" as const,`);
|
||||
if (item.icon) {
|
||||
lines.push(` icon: "${item.icon}",`);
|
||||
}
|
||||
lines.push(' }' + (index < items.length - 1 ? ',' : ''));
|
||||
});
|
||||
|
||||
lines.push('];');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Main function with enhanced features
|
||||
*/
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length === 0) {
|
||||
console.log('Enhanced WoW Item Data Fetcher');
|
||||
console.log('Usage:');
|
||||
console.log(' npm run fetch-items-enhanced [options] [item1] [item2] ...');
|
||||
console.log('');
|
||||
console.log('Options:');
|
||||
console.log(' --existing Fetch data for all items in drop files');
|
||||
console.log(' --validate Compare fetched data with existing project data');
|
||||
console.log(' --generate <name> Generate TypeScript code for new source');
|
||||
console.log(' --blizzard-only Use only Blizzard API');
|
||||
console.log(' --wowhead-only Use only Wowhead API');
|
||||
console.log('');
|
||||
console.log('Examples:');
|
||||
console.log(' npm run fetch-items-enhanced 242494 242495 242481');
|
||||
console.log(' npm run fetch-items-enhanced --existing --validate');
|
||||
console.log(' npm run fetch-items-enhanced --generate "new-dungeon" 123456 123457');
|
||||
return;
|
||||
}
|
||||
|
||||
let itemIds = [];
|
||||
let options = {
|
||||
existing: false,
|
||||
validate: false,
|
||||
generate: null,
|
||||
blizzardOnly: false,
|
||||
wowheadOnly: false
|
||||
};
|
||||
|
||||
// Parse arguments
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
|
||||
if (arg === '--existing') {
|
||||
options.existing = true;
|
||||
} else if (arg === '--validate') {
|
||||
options.validate = true;
|
||||
} else if (arg === '--generate') {
|
||||
options.generate = args[++i];
|
||||
} else if (arg === '--blizzard-only') {
|
||||
options.blizzardOnly = true;
|
||||
} else if (arg === '--wowhead-only') {
|
||||
options.wowheadOnly = true;
|
||||
} else if (!isNaN(parseInt(arg))) {
|
||||
itemIds.push(parseInt(arg));
|
||||
}
|
||||
}
|
||||
|
||||
// Get item IDs
|
||||
if (options.existing) {
|
||||
console.log('Fetching data for existing items in drop files...');
|
||||
const existingIds = await getExistingItemIds();
|
||||
itemIds = [...new Set([...itemIds, ...existingIds])];
|
||||
console.log(`Found ${existingIds.length} existing items`);
|
||||
}
|
||||
|
||||
if (itemIds.length === 0) {
|
||||
console.log('No valid item IDs provided');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Processing ${itemIds.length} items...`);
|
||||
|
||||
// Fetch items with rate limiting
|
||||
const results = [];
|
||||
for (let i = 0; i < itemIds.length; i++) {
|
||||
const itemId = itemIds[i];
|
||||
console.log(`\nProcessing item ${itemId} (${i + 1}/${itemIds.length})`);
|
||||
|
||||
let result;
|
||||
if (options.blizzardOnly) {
|
||||
const blizzardData = await fetchFromBlizzard(itemId);
|
||||
result = { wowhead: null, blizzard: blizzardData, combined: blizzardData };
|
||||
} else if (options.wowheadOnly) {
|
||||
const wowheadData = await fetchFromWowheadJson(itemId) || await fetchFromWowhead(itemId);
|
||||
result = { wowhead: wowheadData, blizzard: null, combined: wowheadData };
|
||||
} else {
|
||||
result = await fetchItemFromAllSources(itemId);
|
||||
}
|
||||
|
||||
if (result.combined) {
|
||||
results.push(result);
|
||||
console.log(`✓ ${result.combined.name} (${result.combined.slot || 'unknown slot'}) [${result.combined.sources.join(', ')}]`);
|
||||
} else {
|
||||
console.log(`✗ Failed to fetch data for item ${itemId}`);
|
||||
}
|
||||
|
||||
// Rate limiting
|
||||
if (i < itemIds.length - 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
// Save detailed results
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
await fs.writeFile(
|
||||
`fetched-items-detailed-${timestamp}.json`,
|
||||
JSON.stringify(results, null, 2)
|
||||
);
|
||||
|
||||
// Validation
|
||||
if (options.validate && options.existing) {
|
||||
console.log('\nValidating against existing project data...');
|
||||
const existingItems = [];
|
||||
// This would need to parse the existing TypeScript files
|
||||
// For now, just show what we fetched
|
||||
|
||||
const validation = validateItemData(results, existingItems);
|
||||
console.log(`\nValidation Results:`);
|
||||
console.log(` Matches: ${validation.matches.length}`);
|
||||
console.log(` Mismatches: ${validation.mismatches.length}`);
|
||||
console.log(` New items: ${validation.newItems.length}`);
|
||||
|
||||
if (validation.mismatches.length > 0) {
|
||||
console.log('\nMismatches found:');
|
||||
validation.mismatches.forEach(mismatch => {
|
||||
console.log(` Item ${mismatch.id}: ${mismatch.issues.join(', ')}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Generate TypeScript code
|
||||
if (options.generate) {
|
||||
const validItems = results
|
||||
.map(r => r.combined)
|
||||
.filter(item => item.name && item.slot);
|
||||
|
||||
const tsCode = generateTypeScriptCode(validItems, options.generate);
|
||||
const filename = `${options.generate}.ts`;
|
||||
|
||||
await fs.writeFile(filename, tsCode);
|
||||
console.log(`\nGenerated TypeScript code in ${filename}`);
|
||||
}
|
||||
|
||||
// Summary
|
||||
console.log(`\nSummary:`);
|
||||
console.log(`Successfully fetched: ${results.length}/${itemIds.length} items`);
|
||||
|
||||
const slotCounts = {};
|
||||
results.forEach(result => {
|
||||
const slot = result.combined.slot || 'unknown';
|
||||
slotCounts[slot] = (slotCounts[slot] || 0) + 1;
|
||||
});
|
||||
|
||||
console.log('\nSlot distribution:');
|
||||
Object.entries(slotCounts).forEach(([slot, count]) => {
|
||||
console.log(` ${slot}: ${count}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch(console.error);
|
||||
}
|
||||
|
||||
export { fetchItemFromAllSources, combineItemData, validateItemData, generateTypeScriptCode };
|
||||
Reference in New Issue
Block a user