diff --git a/sources/academy/webscraping/scraping_basics_javascript/exercises/test.bats b/sources/academy/webscraping/scraping_basics_javascript/exercises/test.bats index d187a5b911..146d5fb216 100644 --- a/sources/academy/webscraping/scraping_basics_javascript/exercises/test.bats +++ b/sources/academy/webscraping/scraping_basics_javascript/exercises/test.bats @@ -1,6 +1,6 @@ setup_file() { cd "$BATS_TEST_DIRNAME" - npm init --yes + echo '{"name":"exercises","private":true}' > package.json npm install cheerio crawlee } @@ -8,6 +8,19 @@ teardown() { rm -rf products.json storage dataset.json } +# Like bats' run, but retries up to 3 times if the command fails with HTTP 429 +run_retry() { + local attempt + for attempt in 1 2 3; do + run "$@" + if (( status == 0 )) || [[ "$output" != *429* ]]; then + return 0 + fi + echo "Attempt $attempt got HTTP 429, retrying..." >&2 + sleep $(( attempt * 10 )) + done +} + teardown_file() { rm -rf node_modules package.json package-lock.json } @@ -27,25 +40,25 @@ teardown_file() { } @test "outputs the HTML with Star Wars products" { - run node lego.mjs + run_retry node lego.mjs [[ "$output" == *"Millennium Falcon"* ]] } @test "counts the number of F1 Academy teams" { - run node f1academy_teams.mjs + run_retry node f1academy_teams.mjs [[ "$output" == "6" ]] } @test "counts the number of F1 Academy drivers" { - run node f1academy_drivers.mjs + run_retry node f1academy_drivers.mjs [[ "$output" == "18" ]] } @test "lists IMO countries" { - run node imo_countries.mjs + run_retry node imo_countries.mjs [[ "$output" == *$'Albania\nLibya\n'* ]] [[ "$output" == *$'\nZimbabwe\nFaroes\n'* ]] @@ -53,7 +66,7 @@ teardown_file() { } @test "lists IMO countries with a single selector" { - run node imo_countries_single_selector.mjs + run_retry node imo_countries_single_selector.mjs [[ "$output" == *$'Albania\nLibya\n'* ]] [[ "$output" == *$'\nZimbabwe\nFaroes\n'* ]] @@ -61,14 +74,14 @@ teardown_file() { } @test "lists Guardian F1 article titles" { - run node guardian_f1_titles.mjs + run_retry node guardian_f1_titles.mjs [[ "$output" == *' F1 '* ]] [[ $(echo "$output" | wc -l) -gt 5 ]] } @test "prints warehouse stock counts" { - run node warehouse_units.mjs + run_retry node warehouse_units.mjs [[ "$output" == *$'JBL Flip 4 Waterproof Portable Bluetooth Speaker | 672\n'* ]] [[ "$output" == *$'Sony XBR-950G BRAVIA 4K HDR Ultra HD TV | 76\n'* ]] @@ -76,7 +89,7 @@ teardown_file() { } @test "prints warehouse stock counts using regex" { - run node warehouse_units_regex.mjs + run_retry node warehouse_units_regex.mjs [[ "$output" == *$'JBL Flip 4 Waterproof Portable Bluetooth Speaker | 672\n'* ]] [[ "$output" == *$'Sony XBR-950G BRAVIA 4K HDR Ultra HD TV | 76\n'* ]] @@ -84,7 +97,7 @@ teardown_file() { } @test "prints Guardian F1 titles with publish dates" { - run node guardian_publish_dates.mjs + run_retry node guardian_publish_dates.mjs [[ "$output" == *' F1 '* ]] [[ "$output" == *' | Mon '* ]] # has info about date, Mondays are very likely @@ -94,27 +107,27 @@ teardown_file() { @test "filters products from JSON" { echo '[{"title":"Premium Speakers","minPrice":75000,"price":75000},{"title":"Budget Headphones","minPrice":25000,"price":25000}]' > products.json - run node process_products_json.mjs + run_retry node process_products_json.mjs [[ "$output" == "{ title: 'Premium Speakers', minPrice: 75000, price: 75000 }" ]] } @test "lists WTA player links" { - run node wta_tennis_links.mjs + run_retry node wta_tennis_links.mjs [[ "$output" == *'https://www.wtatennis.com/players/'* ]] [[ $(echo "$output" | wc -l) -gt 10 ]] } @test "lists Guardian F1 article links" { - run node guardian_f1_links.mjs + run_retry node guardian_f1_links.mjs [[ "$output" == *'https://www.theguardian.com/sport/'* ]] [[ $(echo "$output" | wc -l) -gt 5 ]] } @test "lists WTA player birthplaces" { - run node wta_tennis_players.mjs + run_retry node wta_tennis_players.mjs [[ "$output" == *'https://www.wtatennis.com/players/'* ]] [[ "$output" == *' | '* ]] @@ -123,7 +136,7 @@ teardown_file() { } @test "lists Guardian F1 authors" { - run node guardian_f1_authors.mjs + run_retry node guardian_f1_authors.mjs [[ $(echo "$output" | wc -l) -gt 5 ]] [[ "$output" == *' F1 '* ]] @@ -138,7 +151,7 @@ teardown_file() { } @test "lists JavaScript GitHub repos with the LLM topic" { - run node js_llm_projects.mjs + run_retry node js_llm_projects.mjs (( status == 0 )) [[ $(echo "$output" | wc -l) -eq 37 ]] @@ -151,13 +164,13 @@ teardown_file() { } @test "counts total eurozone population" { - run node eurozone_population.mjs + run_retry node eurozone_population.mjs [[ "$output" -gt 300000000 ]] } @test "scrapes F1 Academy driver details with Crawlee" { - run node crawlee_f1_drivers.mjs + run_retry node crawlee_f1_drivers.mjs (( status == 0 )) [[ -f dataset.json ]] @@ -167,7 +180,7 @@ teardown_file() { } @test "scrapes Netflix user scores with Crawlee" { - run node crawlee_netflix_ratings.mjs + run_retry node crawlee_netflix_ratings.mjs (( status == 0 )) [[ -f dataset.json ]] diff --git a/sources/academy/webscraping/scraping_basics_python/exercises/test.bats b/sources/academy/webscraping/scraping_basics_python/exercises/test.bats index 6d6efa6ea5..a36cdefc02 100644 --- a/sources/academy/webscraping/scraping_basics_python/exercises/test.bats +++ b/sources/academy/webscraping/scraping_basics_python/exercises/test.bats @@ -6,6 +6,19 @@ teardown() { rm -rf products.json storage dataset.json } +# Like bats' run, but retries up to 3 times if the command fails with HTTP 429 +run_retry() { + local attempt + for attempt in 1 2 3; do + run "$@" + if (( status == 0 )) || [[ "$output" != *429* ]]; then + return 0 + fi + echo "Attempt $attempt got HTTP 429, retrying..." >&2 + sleep $(( attempt * 10 )) + done +} + @test "covers all exercise scripts" { local missing missing=0 @@ -21,25 +34,25 @@ teardown() { } @test "outputs the HTML with Star Wars products" { - run uv run -q --with=httpx python lego.py + run_retry uv run -q --with=httpx python lego.py [[ "$output" == *"Millennium Falcon"* ]] } @test "counts the number of F1 Academy teams" { - run uv run -q --with=httpx --with=beautifulsoup4 python f1academy_teams.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python f1academy_teams.py [[ "$output" == "6" ]] } @test "counts the number of F1 Academy drivers" { - run uv run -q --with=httpx --with=beautifulsoup4 python f1academy_drivers.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python f1academy_drivers.py [[ "$output" == "18" ]] } @test "lists IMO countries" { - run uv run -q --with=httpx --with=beautifulsoup4 python imo_countries.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python imo_countries.py [[ "$output" == *$'Albania\nLibya\n'* ]] [[ "$output" == *$'\nZimbabwe\nFaroes\n'* ]] @@ -47,7 +60,7 @@ teardown() { } @test "lists IMO countries with a single selector" { - run uv run -q --with=httpx --with=beautifulsoup4 python imo_countries_single_selector.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python imo_countries_single_selector.py [[ "$output" == *$'Albania\nLibya\n'* ]] [[ "$output" == *$'\nZimbabwe\nFaroes\n'* ]] @@ -55,14 +68,14 @@ teardown() { } @test "lists Guardian F1 article titles" { - run uv run -q --with=httpx --with=beautifulsoup4 python guardian_f1_titles.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python guardian_f1_titles.py [[ "$output" == *' F1 '* ]] [[ $(echo "$output" | wc -l) -gt 5 ]] } @test "prints warehouse stock counts" { - run uv run -q --with=httpx --with=beautifulsoup4 python warehouse_units.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python warehouse_units.py [[ "$output" == *$'JBL Flip 4 Waterproof Portable Bluetooth Speaker | 672\n'* ]] [[ "$output" == *$'Sony XBR-950G BRAVIA 4K HDR Ultra HD TV | 76\n'* ]] @@ -70,7 +83,7 @@ teardown() { } @test "prints warehouse stock counts using regex" { - run uv run -q --with=httpx --with=beautifulsoup4 python warehouse_units_regex.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python warehouse_units_regex.py [[ "$output" == *$'JBL Flip 4 Waterproof Portable Bluetooth Speaker | 672\n'* ]] [[ "$output" == *$'Sony XBR-950G BRAVIA 4K HDR Ultra HD TV | 76\n'* ]] @@ -78,7 +91,7 @@ teardown() { } @test "prints Guardian F1 titles with publish dates" { - run uv run -q --with=httpx --with=beautifulsoup4 python guardian_publish_dates.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python guardian_publish_dates.py [[ "$output" == *' F1 '* ]] [[ "$output" == *' | Mon '* ]] # has info about date, Mondays are very likely @@ -88,27 +101,27 @@ teardown() { @test "filters products from JSON" { echo '[{"title":"Premium Speakers","min_price":75000,"price":75000},{"title":"Budget Headphones","min_price":25000,"price":25000}]' > products.json - run uv run python process_products_json.py + run_retry uv run python process_products_json.py [[ "$output" == "{'title': 'Premium Speakers', 'min_price': 75000, 'price': 75000}" ]] } @test "lists WTA player links" { - run uv run -q --with=httpx --with=beautifulsoup4 python wta_tennis_links.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python wta_tennis_links.py [[ "$output" == *'https://www.wtatennis.com/players/'* ]] [[ $(echo "$output" | wc -l) -gt 10 ]] } @test "lists Guardian F1 article links" { - run uv run -q --with=httpx --with=beautifulsoup4 python guardian_f1_links.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python guardian_f1_links.py [[ "$output" == *'https://www.theguardian.com/sport/'* ]] [[ $(echo "$output" | wc -l) -gt 5 ]] } @test "lists WTA player birthplaces" { - run uv run -q --with=httpx --with=beautifulsoup4 python wta_tennis_players.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python wta_tennis_players.py [[ "$output" == *'https://www.wtatennis.com/players/'* ]] [[ "$output" == *' | '* ]] @@ -117,7 +130,7 @@ teardown() { } @test "lists Guardian F1 authors" { - run uv run -q --with=httpx --with=beautifulsoup4 python guardian_f1_authors.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python guardian_f1_authors.py [[ $(echo "$output" | wc -l) -gt 5 ]] [[ "$output" == *' F1 '* ]] @@ -132,7 +145,7 @@ teardown() { } @test "lists Python database jobs" { - run uv run -q --with=httpx --with=beautifulsoup4 python python_jobs_database.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python python_jobs_database.py [[ "$output" == *"'title': '"* ]] [[ "$output" == *"'company': '"* ]] @@ -141,13 +154,13 @@ teardown() { } @test "counts total eurozone population" { - run uv run -q --with=httpx --with=beautifulsoup4 python eurozone_population.py + run_retry uv run -q --with=httpx --with=beautifulsoup4 python eurozone_population.py [[ "$output" -gt 300000000 ]] } @test "scrapes F1 Academy driver details with Crawlee" { - run uv run -q --with=crawlee[beautifulsoup] python crawlee_f1_drivers.py + run_retry uv run -q --with=crawlee[beautifulsoup] python crawlee_f1_drivers.py (( status == 0 )) [[ -f dataset.json ]] @@ -157,7 +170,7 @@ teardown() { } @test "scrapes Netflix user scores with Crawlee" { - run uv run -q --with=crawlee[beautifulsoup] python crawlee_netflix_ratings.py + run_retry uv run -q --with=crawlee[beautifulsoup] python crawlee_netflix_ratings.py (( status == 0 )) [[ -f dataset.json ]]