From 40e0dbe189620400078b43f7d299090d14302e82 Mon Sep 17 00:00:00 2001 From: Wyatt Walter Date: Thu, 11 Jun 2026 10:19:34 -0500 Subject: [PATCH] ci: distribute SUT image via GHCR instead of Actions cache (#41889) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description The `build-docker-image` workflow currently distributes the system-under-test image to test jobs by saving it to a gzipped tarball (single-threaded, ~4 minutes) and storing it in the Actions cache, which every test shard then restores as a single blob (~3 minutes each, worse under fan-out contention since the restores can't parallelize). This switches distribution to GHCR: the build job pushes the image to `ghcr.io/-citest:run-`, and test jobs pull it from there. Registry pulls are layer-parallel and CDN-backed. Measured on the EE repo across a 20-shard run: median image acquisition dropped from ~218s to 74s per shard, and the build job no longer spends ~4 minutes on `docker save | gzip`. Changes: - `build-docker-image.yml`: push the built `cicontainer` image to GHCR using `GITHUB_TOKEN`; remove the tar/gzip/cache-save steps. - `ci-test-custom-script.yml`, `ci-test-playwright.yml`: pull the image from GHCR and tag it back to `cicontainer`, so downstream `docker run` references are unchanged. - `ci-test-limited.yml`, `ci-test-limited-with-count.yml`: same, honoring `previous-workflow-run-id` for the image-reuse path (`run-` tag). Registry tags outlive Actions cache eviction, so reuse becomes more reliable, bounded by the retention window below. - `ghcr-citest-retention.yml` (new): daily prune of `run-*` package versions older than 7 days (each is ~2 GB). Supports manual dispatch with a dry-run flag. Notes: - The GHCR package is created on first push and defaults to private, linked to this repository. Pulls inside Actions use `GITHUB_TOKEN`; no new secrets or permissions are required. - Fork-PR paths are unaffected: fork-triggered `pull_request` events never reach the build job (existing gate), and the `/ok-to-test` flow runs via `repository_dispatch` in base-repo context. - A `/ci-test-limit` rerun referencing a run that predates this change will not find a registry tag for it; this resolves itself as new runs accumulate. Ref: APP-15259 ## Validation - After merge, run `/ci-test-limit` and `/ci-test-limit-count` on a test PR, including one rerun that exercises `previous-workflow-run-id`. - Dispatch `ghcr-citest-retention.yml` with `dry-run: true` to confirm the token can enumerate and would prune package versions before the first scheduled run. 🤖 Generated with [Claude Code](https://claude.com/claude-code) ## Summary by CodeRabbit * **Chores** * CI now pushes and pulls test container images via the GitHub Container Registry instead of caching/loading tarball artifacts. * Test jobs retrieve run-specific image tags so downstream steps pull images by tag. * Added a scheduled/manual cleanup workflow to prune old test image versions in the registry with configurable retention and dry-run options. > [!WARNING] > Tests have not run on the HEAD a55f67e897d439a9e5a02d611f0b33bbbc6ccc86 yet >
Wed, 10 Jun 2026 20:11:30 UTC --------- Co-authored-by: Claude Fable 5 --- .github/workflows/build-docker-image.yml | 23 ++++----- .github/workflows/ci-test-custom-script.yml | 17 +++---- .../workflows/ci-test-limited-with-count.yml | 31 +++++------ .github/workflows/ci-test-limited.yml | 31 +++++------ .github/workflows/ci-test-playwright.yml | 27 +++++----- .github/workflows/ghcr-citest-retention.yml | 51 +++++++++++++++++++ 6 files changed, 106 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/ghcr-citest-retention.yml diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 54abaccbb208..d39804a027c5 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -120,21 +120,16 @@ jobs: args+=(--label "org.opencontainers.image.version=${{ steps.info_json.outputs.version }}") docker build -t cicontainer "${args[@]}" . - # Saving the docker image to tar file - - name: Save Docker image to tar file + # Push the SUT image to GHCR so test jobs can pull it with layer-level + # parallelism instead of restoring a single-blob tarball from the Actions + # cache. Old run- tags are pruned by ghcr-citest-retention.yml. + - name: Push docker image to GHCR + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }}-citest:run-${{ github.run_id }} run: | - docker image ls --all --no-trunc --format '{{.Repository}},{{.ID}}' \ - | grep -v cicontainer \ - | cut -d, -f2 \ - | xargs -r docker rmi || true - docker save cicontainer -o cicontainer.tar - gzip cicontainer.tar - - - name: Cache docker image - uses: actions/cache/save@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{github.run_id}} + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker tag cicontainer "$GHCR_IMAGE" + docker push "$GHCR_IMAGE" - name: Save the status of the run run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result diff --git a/.github/workflows/ci-test-custom-script.yml b/.github/workflows/ci-test-custom-script.yml index 5c12c4b0f68b..d81ccf937ec5 100644 --- a/.github/workflows/ci-test-custom-script.yml +++ b/.github/workflows/ci-test-custom-script.yml @@ -124,16 +124,15 @@ jobs: - name: cat run_result run: echo ${{ steps.run_result.outputs.run_result }} - - name: Restore the docker image cache - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{github.run_id}} - - - name: Load Docker image from tar file + # The build job pushed the SUT image to GHCR tagged with this run's id; + # pull it from there (layer-parallel, no single-blob cache bottleneck). + - name: Load Docker image + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }}-citest:run-${{ github.run_id }} run: | - gunzip cicontainer.tar.gz - docker load -i cicontainer.tar + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$GHCR_IMAGE" + docker tag "$GHCR_IMAGE" cicontainer - name: Create folder if: steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/ci-test-limited-with-count.yml b/.github/workflows/ci-test-limited-with-count.yml index e91f8a335178..6c8e23606b3c 100644 --- a/.github/workflows/ci-test-limited-with-count.yml +++ b/.github/workflows/ci-test-limited-with-count.yml @@ -204,26 +204,19 @@ jobs: echo "specs_to_run=$specs_to_run" >> $GITHUB_ENV - # In case of run-id provided download the artifact from the previous run - - name: Download Docker image artifact - if: inputs.previous-workflow-run-id != 0 - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{ inputs.previous-workflow-run-id }} - - # In case of run-id is 0 download the artifact from the current run - - name: Download Docker image artifact - if: inputs.previous-workflow-run-id == 0 - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{github.run_id}} - - - name: Load Docker image from tar file + # The build job pushed the SUT image to GHCR tagged with its run id; + # when previous-workflow-run-id is set, reuse the image built by that + # earlier run instead. + - name: Load Docker image run: | - gunzip cicontainer.tar.gz - docker load -i cicontainer.tar + image_run_id='${{ inputs.previous-workflow-run-id }}' + if [[ "$image_run_id" == '0' ]]; then + image_run_id='${{ github.run_id }}' + fi + ghcr_image="ghcr.io/${{ github.repository }}-citest:run-$image_run_id" + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$ghcr_image" + docker tag "$ghcr_image" cicontainer - name: Create folder if: steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/ci-test-limited.yml b/.github/workflows/ci-test-limited.yml index c965f5568412..c0d6672f6957 100644 --- a/.github/workflows/ci-test-limited.yml +++ b/.github/workflows/ci-test-limited.yml @@ -115,26 +115,19 @@ jobs: specs_to_run=${specs_to_run#,} echo "specs_to_run=$specs_to_run" >> $GITHUB_ENV - # In case of run-id provided download the artifact from the previous run - - name: Download Docker image artifact - if: inputs.previous-workflow-run-id != 0 - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{ inputs.previous-workflow-run-id }} - - # In case of run-id is 0 download the artifact from the current run - - name: Download Docker image artifact - if: inputs.previous-workflow-run-id == 0 - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{github.run_id}} - - - name: Load Docker image from tar file + # The build job pushed the SUT image to GHCR tagged with its run id; + # when previous-workflow-run-id is set, reuse the image built by that + # earlier run instead. + - name: Load Docker image run: | - gunzip cicontainer.tar.gz - docker load -i cicontainer.tar + image_run_id='${{ inputs.previous-workflow-run-id }}' + if [[ "$image_run_id" == '0' ]]; then + image_run_id='${{ github.run_id }}' + fi + ghcr_image="ghcr.io/${{ github.repository }}-citest:run-$image_run_id" + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$ghcr_image" + docker tag "$ghcr_image" cicontainer - name: Create folder if: steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/ci-test-playwright.yml b/.github/workflows/ci-test-playwright.yml index 3a63999e9dd4..9348eb579539 100644 --- a/.github/workflows/ci-test-playwright.yml +++ b/.github/workflows/ci-test-playwright.yml @@ -91,27 +91,28 @@ jobs: password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Get Docker image id + env: + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} run: | - if [[ '${{ inputs.docker_image_name }}' != '' ]]; then - echo 'docker_container_name=${{ inputs.docker_image_name }}' >> "$GITHUB_ENV" + if [[ -n "$DOCKER_IMAGE_NAME" ]]; then + echo "docker_container_name=$DOCKER_IMAGE_NAME" >> "$GITHUB_ENV" else echo 'docker_container_name=cicontainer' >> "$GITHUB_ENV" fi - - name: Restore Docker image cache - if: inputs.docker_image_name == '' - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{ github.run_id }} - + # The build job pushed the SUT image to GHCR tagged with this run's id; + # pull it from there (layer-parallel, no single-blob cache bottleneck). - name: Load Docker image + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }}-citest:run-${{ github.run_id }} + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} run: | - if [[ '${{ inputs.docker_image_name }}' == '' ]]; then - gunzip cicontainer.tar.gz - docker load -i cicontainer.tar + if [[ -z "$DOCKER_IMAGE_NAME" ]]; then + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$GHCR_IMAGE" + docker tag "$GHCR_IMAGE" cicontainer else - docker pull ${{ env.docker_container_name }} + docker pull "$DOCKER_IMAGE_NAME" fi - name: Create Appsmith stacks folder diff --git a/.github/workflows/ghcr-citest-retention.yml b/.github/workflows/ghcr-citest-retention.yml new file mode 100644 index 000000000000..9b5881736a02 --- /dev/null +++ b/.github/workflows/ghcr-citest-retention.yml @@ -0,0 +1,51 @@ +name: GHCR citest image retention + +# The build-docker-image workflow pushes a ~2 GB SUT image to GHCR tagged +# run- on every run. Test jobs only pull images from the current run, +# or (for /ci-test-limit reruns) from a recent previous run, so anything older +# than the retention window is dead weight. This prunes those versions daily. + +on: + schedule: + - cron: "47 3 * * *" + workflow_dispatch: + inputs: + older-than-days: + description: "Delete run-* versions older than this many days" + required: false + type: string + default: "7" + dry-run: + description: "Only log what would be deleted" + required: false + type: boolean + default: false + +permissions: + packages: write + +jobs: + prune: + runs-on: ubuntu-latest + steps: + - name: Delete citest package versions past the retention window + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OLDER_THAN_DAYS: ${{ inputs.older-than-days || '7' }} + DRY_RUN: ${{ inputs.dry-run && 'true' || 'false' }} + run: | + set -o errexit -o nounset -o pipefail + PACKAGE_PATH="/orgs/${GITHUB_REPOSITORY_OWNER}/packages/container/${GITHUB_REPOSITORY#*/}-citest" + cutoff=$(( $(date +%s) - OLDER_THAN_DAYS * 86400 )) + deleted=0 + while IFS=$'\t' read -r id created tags; do + echo "Deleting version $id (created $created, tags: $tags)" + if [[ "$DRY_RUN" != 'true' ]]; then + gh api --method DELETE "$PACKAGE_PATH/versions/$id" + fi + deleted=$((deleted + 1)) + done < <( + gh api --paginate "$PACKAGE_PATH/versions?per_page=100" \ + --jq ".[] | select((.created_at | fromdateiso8601) < $cutoff) | [.id, .created_at, (.metadata.container.tags | join(\",\"))] | @tsv" + ) + echo "Deleted $deleted version(s) older than $OLDER_THAN_DAYS day(s) (dry-run: $DRY_RUN)"