From 5d4b4b300769423c07f6f55b5f28cab666f514c3 Mon Sep 17 00:00:00 2001 From: Oleksandr_37 Date: Mon, 12 May 2025 21:24:57 +0300 Subject: [PATCH 1/3] Fixed caching logic to fetch all the necessary data --- scripts/download.js | 57 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/scripts/download.js b/scripts/download.js index c273a1f..b0c41cd 100644 --- a/scripts/download.js +++ b/scripts/download.js @@ -47,21 +47,60 @@ async function cacheResources() { const cache = {}; for (const name of resources) { + console.log(`Caching resource: "${name}"`); let url = `${swapiPath}/${name}`; while (url != null) { - const response = await fetch(url, { agent }); - const text = await response.text(); + try { + const response = await fetch(url, { agent }); + const text = await response.text(); - const data = JSON.parse(replaceHttp(text)); + let data = {}; + try { + data = JSON.parse(replaceHttp(text)); + } catch (e) { + console.error(`Error parsing JSON from ${url}:`, e); + break; + } - cache[normalizeUrl(url)] = data; - for (const obj of data.result || data.results || []) { - const itemUrl = obj.url || obj.properties.url; - cache[normalizeUrl(itemUrl)] = obj; - } + const properResults = await Promise.allSettled( + (data.result || data.results || []).map(async obj => { + const itemUrl = obj.url || obj.properties.url; + try { + const response = await fetch(itemUrl, { agent }); + const itemText = await response.text(); + + let itemData = {}; + try { + itemData = JSON.parse(replaceHttp(itemText)); + } catch (e) { + console.error(`Error parsing JSON from ${itemUrl}:`, e); + } + + const fullObjectData = { + ...obj, + ...((itemData.result || itemData.results || {}).properties || + {}), + }; + + cache[normalizeUrl(itemUrl)] = fullObjectData; - url = data.next ? data.next.replace('http:', 'https:') : null; + return fullObjectData; + } catch (e) { + console.error(`Error fetching item from ${itemUrl}:`, e); + return obj; // Return the original object if fetching fails + } + }), + ); + + data.results = properResults; + cache[normalizeUrl(url)] = data; + + url = data.next ? data.next.replace('http:', 'https:') : null; + } catch (e) { + console.error(`Error fetching resource from ${url}:`, e); + break; + } } } From 8b929f7723d8ec78f1e7c7c8f21adfed9e30ca8d Mon Sep 17 00:00:00 2001 From: Oleksandr_37 Date: Mon, 12 May 2025 22:28:08 +0300 Subject: [PATCH 2/3] Fixed getting values from promises results --- scripts/download.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/download.js b/scripts/download.js index b0c41cd..1af512b 100644 --- a/scripts/download.js +++ b/scripts/download.js @@ -17,8 +17,8 @@ const resources = [ 'starships', 'vehicles', 'species', - 'planets', 'films', + 'planets', ]; function replaceHttp(url) { @@ -93,7 +93,7 @@ async function cacheResources() { }), ); - data.results = properResults; + data.results = properResults.map(r => r.value || {}); cache[normalizeUrl(url)] = data; url = data.next ? data.next.replace('http:', 'https:') : null; @@ -122,7 +122,7 @@ if (!existsSync(outfile)) { writeFileSync(outfile, data, 'utf-8'); console.log('Cached!'); }) - .catch(function (err) { + .catch(function(err) { console.error(err); process.exit(1); }); From 59776ab22b1056633247fef3cb405975f5ea4646 Mon Sep 17 00:00:00 2001 From: Oleksandr_37 Date: Mon, 12 May 2025 22:29:21 +0300 Subject: [PATCH 3/3] Removed redundant changes --- scripts/download.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/download.js b/scripts/download.js index 1af512b..cefc628 100644 --- a/scripts/download.js +++ b/scripts/download.js @@ -17,8 +17,8 @@ const resources = [ 'starships', 'vehicles', 'species', - 'films', 'planets', + 'films', ]; function replaceHttp(url) { @@ -122,7 +122,7 @@ if (!existsSync(outfile)) { writeFileSync(outfile, data, 'utf-8'); console.log('Cached!'); }) - .catch(function(err) { + .catch(function (err) { console.error(err); process.exit(1); });