Fetches better item data, adds spec selector
This commit is contained in:
313
scripts/fetch-item-data.js
Normal file
313
scripts/fetch-item-data.js
Normal file
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import https from "https";
|
||||
import fs from "fs/promises";
|
||||
import path from "path";
|
||||
|
||||
/**
|
||||
* Fetches item data from Wowhead API
|
||||
* @param {number} itemId - The item ID to fetch
|
||||
* @returns {Promise<Object|null>} Item data or null if failed
|
||||
*/
|
||||
async function fetchFromWowhead(itemId, source) {
|
||||
return new Promise((resolve) => {
|
||||
const url = `https://www.wowhead.com/item=${itemId}&xml`;
|
||||
|
||||
https
|
||||
.get(url, (res) => {
|
||||
let data = "";
|
||||
|
||||
res.on("data", (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on("end", () => {
|
||||
try {
|
||||
// Weirdly, the JSON in the XML has more data than the XML itself or the JSON API.
|
||||
const jsonString = data.match(/<json><!\[CDATA\[(.*)\]\]><\/json>/);
|
||||
const jsonData = JSON.parse(`{${jsonString[1]}}`);
|
||||
|
||||
// Parse the XML response from Wowhead
|
||||
// const nameMatch = data.match(/<name><!\[CDATA\[(.*?)\]\]><\/name>/);
|
||||
const iconMatch = data.match(
|
||||
/<icon displayId="(\d+)">(.*?)<\/icon>/,
|
||||
);
|
||||
const slotMatch = data.match(
|
||||
/<inventorySlot id="(\d+)">(.*?)<\/inventorySlot>/,
|
||||
);
|
||||
|
||||
const name = jsonData.displayName;
|
||||
|
||||
const itemData = {
|
||||
id: itemId,
|
||||
name,
|
||||
icon: iconMatch ? iconMatch[2] : null,
|
||||
slot: mapSlotFromWowhead(jsonData.slot),
|
||||
specs: jsonData.specs || [],
|
||||
source,
|
||||
};
|
||||
resolve(itemData);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Error parsing Wowhead data for item ${itemId}:`,
|
||||
error.message,
|
||||
);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
})
|
||||
.on("error", (error) => {
|
||||
console.error(
|
||||
`Error fetching from Wowhead for item ${itemId}:`,
|
||||
error.message,
|
||||
);
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches item data from Wowhead's JSON API (alternative method)
|
||||
* @param {number} itemId - The item ID to fetch
|
||||
* @returns {Promise<Object|null>} Item data or null if failed
|
||||
*/
|
||||
async function fetchFromWowheadJson(itemId) {
|
||||
return new Promise((resolve) => {
|
||||
const url = `https://nether.wowhead.com/tooltip/item/${itemId}`;
|
||||
|
||||
https
|
||||
.get(url, (res) => {
|
||||
let data = "";
|
||||
|
||||
res.on("data", (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on("end", () => {
|
||||
try {
|
||||
const jsonData = JSON.parse(data);
|
||||
|
||||
if (jsonData && jsonData.name) {
|
||||
const itemData = {
|
||||
id: itemId,
|
||||
name: jsonData.name,
|
||||
icon: jsonData.icon || null,
|
||||
slot: mapSlotFromWowhead(jsonData.slot || 0),
|
||||
};
|
||||
resolve(itemData);
|
||||
} else {
|
||||
resolve(null);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Error parsing Wowhead JSON data for item ${itemId}:`,
|
||||
error.message,
|
||||
);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
})
|
||||
.on("error", (error) => {
|
||||
console.error(
|
||||
`Error fetching from Wowhead JSON for item ${itemId}:`,
|
||||
error.message,
|
||||
);
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps Wowhead slot numbers to our slot names
|
||||
* @param {number} slotId - Wowhead slot ID
|
||||
* @returns {string|null} Our slot name or null
|
||||
*/
|
||||
function mapSlotFromWowhead(slotId) {
|
||||
const slotMap = {
|
||||
1: "head",
|
||||
2: "neck",
|
||||
3: "shoulders",
|
||||
5: "chest",
|
||||
6: "waist",
|
||||
7: "legs",
|
||||
8: "feet",
|
||||
9: "wrist",
|
||||
10: "hands",
|
||||
11: "finger",
|
||||
12: "trinket",
|
||||
13: "1h-weapon",
|
||||
14: "off-hand",
|
||||
15: "back",
|
||||
16: "1h-weapon", // Main hand
|
||||
17: "2h-weapon",
|
||||
21: "1h-weapon", // One-handed
|
||||
22: "off-hand",
|
||||
23: "off-hand", // Held in off-hand
|
||||
25: "1h-weapon", // Thrown
|
||||
};
|
||||
|
||||
return slotMap[slotId] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches multiple items with rate limiting
|
||||
* @param {number[]} itemIds - Array of item IDs to fetch
|
||||
* @param {number} delay - Delay between requests in ms
|
||||
* @returns {Promise<Object[]>} Array of item data
|
||||
*/
|
||||
async function fetchMultipleItems(source, itemIds, delay = 1000) {
|
||||
const results = [];
|
||||
|
||||
console.log(`Fetching data for ${itemIds.length} items...`);
|
||||
|
||||
for (let i = 0; i < itemIds.length; i++) {
|
||||
const itemId = itemIds[i];
|
||||
console.log(`Fetching item ${itemId} (${i + 1}/${itemIds.length})`);
|
||||
|
||||
// Try Wowhead JSON API first, fallback to XML
|
||||
let itemData = await fetchFromWowhead(itemId, source);
|
||||
if (!itemData) {
|
||||
itemData = await fetchFromWowheadJson(itemId, source);
|
||||
}
|
||||
|
||||
if (itemData) {
|
||||
results.push(itemData);
|
||||
console.log(
|
||||
`✓ Found: ${itemData.name} (${itemData.slot || "unknown slot"})`,
|
||||
);
|
||||
} else {
|
||||
console.log(`✗ Failed to fetch data for item ${itemId}`);
|
||||
}
|
||||
|
||||
// Rate limiting
|
||||
if (i < itemIds.length - 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads item IDs from existing drop files
|
||||
* @returns {Promise<number[]>} Array of item IDs
|
||||
*/
|
||||
async function getExistingItemIds() {
|
||||
const dropsDir = path.join(process.cwd(), "src", "lib", "drops");
|
||||
const itemIds = [];
|
||||
|
||||
try {
|
||||
const files = await fs.readdir(dropsDir);
|
||||
|
||||
for (const file of files) {
|
||||
if (file.endsWith(".ts") && file !== "index.ts") {
|
||||
const filePath = path.join(dropsDir, file);
|
||||
const content = await fs.readFile(filePath, "utf-8");
|
||||
|
||||
// Extract item IDs using regex
|
||||
const idMatches = content.match(/id: (\d+) as ItemId/g);
|
||||
if (idMatches) {
|
||||
for (const match of idMatches) {
|
||||
const id = parseInt(match.match(/id: (\d+)/)[1]);
|
||||
itemIds.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error reading existing item IDs:", error.message);
|
||||
}
|
||||
|
||||
return [...new Set(itemIds)]; // Remove duplicates
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves fetched data to a JSON file
|
||||
* @param {Object[]} items - Array of item data
|
||||
* @param {string} filename - Output filename
|
||||
*/
|
||||
async function saveToFile(items, filename = "fetched-items.json") {
|
||||
try {
|
||||
const outputPath = path.join(process.cwd(), filename);
|
||||
await fs.writeFile(outputPath, JSON.stringify(items, null, 2));
|
||||
console.log(`\nSaved ${items.length} items to ${filename}`);
|
||||
} catch (error) {
|
||||
console.error("Error saving file:", error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main function
|
||||
*/
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length === 0) {
|
||||
console.log(
|
||||
"Usage: npm run fetch-items [item1] [item2] ... or npm run fetch-items --existing",
|
||||
);
|
||||
console.log("Examples:");
|
||||
console.log(" npm run fetch-items 242494 242495 242481");
|
||||
console.log(
|
||||
" npm run fetch-items --existing # Fetch data for all items in drop files",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let itemIds = [];
|
||||
let source = null;
|
||||
|
||||
if (args[0] === "--existing") {
|
||||
console.log("Fetching data for existing items in drop files...");
|
||||
itemIds = await getExistingItemIds();
|
||||
console.log(`Found ${itemIds.length} existing items`);
|
||||
} else {
|
||||
source = args[0];
|
||||
itemIds = args
|
||||
.slice(1)
|
||||
.map((arg) => parseInt(arg))
|
||||
.filter((id) => !isNaN(id));
|
||||
}
|
||||
|
||||
if (itemIds.length === 0) {
|
||||
console.log("No valid item IDs provided");
|
||||
return;
|
||||
}
|
||||
|
||||
const items = await fetchMultipleItems(source, itemIds);
|
||||
|
||||
if (items.length > 0) {
|
||||
await saveToFile(items);
|
||||
|
||||
console.log("\nSummary:");
|
||||
console.log(
|
||||
`Successfully fetched: ${items.length}/${itemIds.length} items`,
|
||||
);
|
||||
|
||||
// Show slot distribution
|
||||
const slotCounts = {};
|
||||
items.forEach((item) => {
|
||||
const slot = item.slot || "unknown";
|
||||
slotCounts[slot] = (slotCounts[slot] || 0) + 1;
|
||||
});
|
||||
|
||||
console.log("\nSlot distribution:");
|
||||
Object.entries(slotCounts).forEach(([slot, count]) => {
|
||||
console.log(` ${slot}: ${count}`);
|
||||
});
|
||||
} else {
|
||||
console.log("No items were successfully fetched");
|
||||
}
|
||||
}
|
||||
|
||||
// Run the script
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch(console.error);
|
||||
}
|
||||
|
||||
export {
|
||||
fetchFromWowhead,
|
||||
fetchFromWowheadJson,
|
||||
fetchMultipleItems,
|
||||
getExistingItemIds,
|
||||
};
|
||||
Reference in New Issue
Block a user