From 312f1436d2138ec178d47530c83782305471bcd2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 20:42:29 +0000 Subject: [PATCH 1/3] Stop sharding the gameplay tests twice on CI The runner defaulted its shard to CIRCLE_NODE_INDEX/CIRCLE_NODE_TOTAL, which CircleCI sets on every container as soon as parallelism is above 1. The all-games job therefore cut the run down twice: `--list` printed only this container's quarter of the games, `circleci tests split` split that quarter again, and the run sharded what was left once more. With 4 containers each one ended up testing a single game, so 4 of the 45 games with gameplay tests were tested on main instead of all of them. Sharding now only happens when --shard-index/--shard-total are passed explicitly. The CI splits with `circleci tests split`, which balances by the timings recorded by store_test_results; this script runs what it is given. That also explains the "No timing found" lines: with only four games ever run, there were no timings for any of the others. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01M2jc7PVAvmMmirAQude2v1 --- scripts/run-gameplay-tests.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/run-gameplay-tests.js b/scripts/run-gameplay-tests.js index 8da4c2e84..5025b5e3a 100644 --- a/scripts/run-gameplay-tests.js +++ b/scripts/run-gameplay-tests.js @@ -19,8 +19,10 @@ * `circleci tests split`). * --list Print the project files that would be tested, * one per line, and exit. Nothing is run. - * --shard-index=0 Test only a slice of the projects. Defaults to - * --shard-total=1 CIRCLE_NODE_INDEX / CIRCLE_NODE_TOTAL. + * --shard-index=0 Test only one slice of the projects, for + * --shard-total=1 splitting a run by hand. NOT used by the CI, + * which splits with `circleci tests split` (and + * balances by timing) - see the note below. * --gdevelop-branch=master Branch of GDevelop to take the build from. * --gdevelop-version=5.6.277 Skip reading the version from the branch. * --work-dir=... Where GDevelop is downloaded and extracted @@ -70,10 +72,15 @@ const junitPath = path.resolve( path.join(repositoryPath, 'gameplay-tests-results/results.xml') ); const timeoutMs = Number(args['timeout-ms']) || 15 * 60 * 1000; -const shardTotal = - Number(args['shard-total'] || process.env.CIRCLE_NODE_TOTAL) || 1; -const shardIndex = - Number(args['shard-index'] || process.env.CIRCLE_NODE_INDEX) || 0; +// Sharding only happens when it is asked for explicitly. It used to default +// to CIRCLE_NODE_INDEX/CIRCLE_NODE_TOTAL, which quietly cut the run down +// twice on CI: `--list` returned only this container's quarter of the games, +// `circleci tests split` then split that quarter again, and the run sharded +// what was left once more - so each of 4 containers tested a single game +// instead of a quarter of them. The CI splits with `circleci tests split`, +// which balances by recorded timings; this script just runs what it is given. +const shardTotal = Number(args['shard-total']) || 1; +const shardIndex = Number(args['shard-index']) || 0; /** * Print an informational message. With `--list`, stdout is reserved for the From 69353dca6cffcaa2c212b9ad4224f0cf58fafa35 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 20:58:26 +0000 Subject: [PATCH 2/3] Take a screenshot at the end of ten gameplay tests One test per game, for the ones whose result is worth looking at: the tank's target after the shell exploded on it, the target that was shot at in the first person shooter, the car past the finish line, the RTS selection walking to where it was sent, the two 3D boards after a piece was dropped or a tile placed, the RPG scene after the NPC was answered, the first person farming inventory holding the harvested seed, and both platformers once their coins are collected. They are 9 to 17 KB each and the CI already stores them next to the results of the game they belong to. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01M2jc7PVAvmMmirAQude2v1 --- README.md | 7 +++++++ .../starting-3D-platformer/starting-3D-platformer.json | 6 +++++- .../starting-3d-car-racing/starting-3d-car-racing.json | 6 +++++- .../starting-3d-draggable-tiles.json | 6 +++++- .../starting-3d-rts-unit-selection.json | 6 +++++- examples/starting-3d-tank/starting-3d-tank.json | 6 +++++- .../starting-3d-tile-placement.json | 6 +++++- .../starting-first-person-farming.json | 6 +++++- .../starting-first-person-shooter.json | 6 +++++- examples/starting-platformer/starting-platformer.json | 6 +++++- examples/starting-top-down-rpg/starting-top-down-rpg.json | 6 +++++- 11 files changed, 57 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b3e487ded..218a0058e 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,13 @@ took with `harness.takeScreenshot(...)` in a `gameplay-test-screenshots/` folder next to the results — which refer to them by a relative path, so a downloaded results file still points at the right images. +A handful of tests end with a screenshot of what the game looks like once +the thing they check has happened (the tank's target after the shell +exploded on it, the inventory holding the seed that was harvested, the car +past the finish line with the lap counter at 1...). They are a few kilobytes +each and make a run readable at a glance, so they are worth adding to a test +whose result is something you would want to look at. + The tests of every game can also be run on a branch, without waiting for it to land on `main`, by triggering a CircleCI pipeline with the `run-all-gameplay-tests` parameter set to `true`. The number of parallel diff --git a/examples/starting-3D-platformer/starting-3D-platformer.json b/examples/starting-3D-platformer/starting-3D-platformer.json index e440d06cd..8c8e2cce7 100644 --- a/examples/starting-3D-platformer/starting-3D-platformer.json +++ b/examples/starting-3D-platformer/starting-3D-platformer.json @@ -2764,7 +2764,11 @@ "harness.assert(", " coinIds().length < coinsBefore,", " 'The number of coins left in the level went down.'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the player standing where the coin was');" ] } ], diff --git a/examples/starting-3d-car-racing/starting-3d-car-racing.json b/examples/starting-3d-car-racing/starting-3d-car-racing.json index 0008611f8..bc6ff65d8 100644 --- a/examples/starting-3d-car-racing/starting-3d-car-racing.json +++ b/examples/starting-3d-car-racing/starting-3d-car-racing.json @@ -2212,7 +2212,11 @@ " finalProgress.checkpoint === 1,", " 'The next checkpoint to reach is the first one of the new lap (it is ' +", " finalProgress.checkpoint + ').'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the car just past the finish line');" ] } ], diff --git a/examples/starting-3d-draggable-tiles/starting-3d-draggable-tiles.json b/examples/starting-3d-draggable-tiles/starting-3d-draggable-tiles.json index 4e74700f3..caea6faa4 100644 --- a/examples/starting-3d-draggable-tiles/starting-3d-draggable-tiles.json +++ b/examples/starting-3d-draggable-tiles/starting-3d-draggable-tiles.json @@ -1308,7 +1308,11 @@ " 'The piece is dropped on the cell it was dragged to (it is at ' +", " Math.round(dropped.x) + ',' + Math.round(dropped.y) +", " ', expected ' + targetX + ',' + targetY + ').'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the board with the piece dropped on its new cell');" ] }, { diff --git a/examples/starting-3d-rts-unit-selection/starting-3d-rts-unit-selection.json b/examples/starting-3d-rts-unit-selection/starting-3d-rts-unit-selection.json index 58d1ae20d..3e79f5933 100644 --- a/examples/starting-3d-rts-unit-selection/starting-3d-rts-unit-selection.json +++ b/examples/starting-3d-rts-unit-selection/starting-3d-rts-unit-selection.json @@ -1771,7 +1771,11 @@ " unitsThatMoved === positionsBeforeOrder.size,", " 'Every unit of the selection walks to where the group was ordered (' +", " unitsThatMoved + ' of ' + positionsBeforeOrder.size + ' moved).'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the whole selection walking to where it was sent');" ] } ], diff --git a/examples/starting-3d-tank/starting-3d-tank.json b/examples/starting-3d-tank/starting-3d-tank.json index f601aba51..f1d182861 100644 --- a/examples/starting-3d-tank/starting-3d-tank.json +++ b/examples/starting-3d-tank/starting-3d-tank.json @@ -2472,7 +2472,11 @@ "harness.assert(", " blownAway,", " 'The explosion blows the target away (it moved ' + Math.round(knockback) + 'px).'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the target after the shell exploded on it');" ] } ], diff --git a/examples/starting-3d-tile-placement/starting-3d-tile-placement.json b/examples/starting-3d-tile-placement/starting-3d-tile-placement.json index 43aef52bf..3aa2c67cf 100644 --- a/examples/starting-3d-tile-placement/starting-3d-tile-placement.json +++ b/examples/starting-3d-tile-placement/starting-3d-tile-placement.json @@ -1647,7 +1647,11 @@ " placedTiles() === 1,", " 'Clicking a cell that already holds a tile does not place another one (' +", " placedTiles() + ' tiles on the board).'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the board with the tile placed on it');" ] } ], diff --git a/examples/starting-first-person-farming/starting-first-person-farming.json b/examples/starting-first-person-farming/starting-first-person-farming.json index 01536b2f5..69e2ac67f 100644 --- a/examples/starting-first-person-farming/starting-first-person-farming.json +++ b/examples/starting-first-person-farming/starting-first-person-farming.json @@ -6511,7 +6511,11 @@ " getPlayer().rotationY > 2,", " 'The view really had to be lowered onto the patch (it ended at ' +", " getPlayer().rotationY.toFixed(1) + ' degrees).'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the inventory holding the seed that was harvested');" ] } ], diff --git a/examples/starting-first-person-shooter/starting-first-person-shooter.json b/examples/starting-first-person-shooter/starting-first-person-shooter.json index 521da3b75..7777e69c0 100644 --- a/examples/starting-first-person-shooter/starting-first-person-shooter.json +++ b/examples/starting-first-person-shooter/starting-first-person-shooter.json @@ -2831,7 +2831,11 @@ "harness.assert(", " knockedOver,", " 'Being shot knocks the target over (it moved ' + pushed.toFixed(1) + ' units).'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the target that was shot at');" ] } ], diff --git a/examples/starting-platformer/starting-platformer.json b/examples/starting-platformer/starting-platformer.json index f16e55659..f6fe1e2c6 100644 --- a/examples/starting-platformer/starting-platformer.json +++ b/examples/starting-platformer/starting-platformer.json @@ -1292,7 +1292,11 @@ "harness.assert(", " remaining.length === coinsBefore - targetIds.length,", " 'Only the coins the player ran into were collected.'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the level once every coin has been collected');" ] } ], diff --git a/examples/starting-top-down-rpg/starting-top-down-rpg.json b/examples/starting-top-down-rpg/starting-top-down-rpg.json index 258a5863a..c9b516181 100644 --- a/examples/starting-top-down-rpg/starting-top-down-rpg.json +++ b/examples/starting-top-down-rpg/starting-top-down-rpg.json @@ -1679,7 +1679,11 @@ "harness.assert(", " !harness.getObjects('NPC').some((one) => one.id === npc.id),", " 'The NPC that was talked to is the one that left.'", - ");" + ");", + "", + "// A picture of what the game looks like once this has happened, kept with", + "// the test results.", + "await harness.takeScreenshot('the scene after the NPC was answered');" ] } ], From 75b98aa5fd4f18109e54a1c75e13fa5dec5531a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 21:10:59 +0000 Subject: [PATCH 3/3] Say which file made the changed-games run test everything The message only said "the gameplay tests runner or the CI configuration changed", leaving the reader to work out which of the six files it was, and why a Pull Request that touches no game was testing all 45 of them. It now names the file and says what it implies. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01M2jc7PVAvmMmirAQude2v1 --- scripts/lib/ChangedProjectFiles.js | 6 +++--- scripts/run-gameplay-tests.js | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/lib/ChangedProjectFiles.js b/scripts/lib/ChangedProjectFiles.js index 55d4d9ea1..0cdb9330d 100644 --- a/scripts/lib/ChangedProjectFiles.js +++ b/scripts/lib/ChangedProjectFiles.js @@ -35,7 +35,7 @@ const runGitCommand = (command) => { * the base branch in the meantime are not reported as changes. * @param {Object} options * @param {string} options.baseRef For example `origin/main`. - * @returns {{ changedFiles: string[], requiresFullRun: boolean } | null} Null + * @returns {{ changedFiles: string[], filesRequiringAFullRun: string[] } | null} Null * when the changed files could not be determined (the caller should then * test everything rather than nothing). */ @@ -65,10 +65,10 @@ const findChangedFiles = ({ baseRef }) => { } const changedFiles = diffOutput.split('\n').filter(Boolean); - const requiresFullRun = changedFiles.some((changedFile) => + const filesRequiringAFullRun = changedFiles.filter((changedFile) => FILES_REQUIRING_A_FULL_RUN.includes(changedFile) ); - return { changedFiles, requiresFullRun }; + return { changedFiles, filesRequiringAFullRun }; }; /** diff --git a/scripts/run-gameplay-tests.js b/scripts/run-gameplay-tests.js index 5025b5e3a..15d2e6ca2 100644 --- a/scripts/run-gameplay-tests.js +++ b/scripts/run-gameplay-tests.js @@ -130,9 +130,12 @@ const getRestrictedProjectFiles = () => { ); return null; } - if (changes.requiresFullRun) { + if (changes.filesRequiringAFullRun.length > 0) { + // A change to the runner or to the CI can break any game, so testing only + // the games whose project file changed would say nothing about it. log( - 'ℹ️ The gameplay tests runner or the CI configuration changed: testing every game.' + `ℹ️ Testing every game: ${changes.filesRequiringAFullRun.join(', ')} ` + + 'changed, and that affects how every game is tested.' ); return null; }