diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 54abaccbb20..d39804a027c 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -120,21 +120,16 @@ jobs: args+=(--label "org.opencontainers.image.version=${{ steps.info_json.outputs.version }}") docker build -t cicontainer "${args[@]}" . - # Saving the docker image to tar file - - name: Save Docker image to tar file + # Push the SUT image to GHCR so test jobs can pull it with layer-level + # parallelism instead of restoring a single-blob tarball from the Actions + # cache. Old run- tags are pruned by ghcr-citest-retention.yml. + - name: Push docker image to GHCR + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }}-citest:run-${{ github.run_id }} run: | - docker image ls --all --no-trunc --format '{{.Repository}},{{.ID}}' \ - | grep -v cicontainer \ - | cut -d, -f2 \ - | xargs -r docker rmi || true - docker save cicontainer -o cicontainer.tar - gzip cicontainer.tar - - - name: Cache docker image - uses: actions/cache/save@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{github.run_id}} + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker tag cicontainer "$GHCR_IMAGE" + docker push "$GHCR_IMAGE" - name: Save the status of the run run: echo "run_result=success" >> $GITHUB_OUTPUT > ~/run_result diff --git a/.github/workflows/ci-test-custom-script.yml b/.github/workflows/ci-test-custom-script.yml index 5c12c4b0f68..d81ccf937ec 100644 --- a/.github/workflows/ci-test-custom-script.yml +++ b/.github/workflows/ci-test-custom-script.yml @@ -124,16 +124,15 @@ jobs: - name: cat run_result run: echo ${{ steps.run_result.outputs.run_result }} - - name: Restore the docker image cache - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{github.run_id}} - - - name: Load Docker image from tar file + # The build job pushed the SUT image to GHCR tagged with this run's id; + # pull it from there (layer-parallel, no single-blob cache bottleneck). + - name: Load Docker image + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }}-citest:run-${{ github.run_id }} run: | - gunzip cicontainer.tar.gz - docker load -i cicontainer.tar + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$GHCR_IMAGE" + docker tag "$GHCR_IMAGE" cicontainer - name: Create folder if: steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/ci-test-limited-with-count.yml b/.github/workflows/ci-test-limited-with-count.yml index e91f8a33517..6c8e23606b3 100644 --- a/.github/workflows/ci-test-limited-with-count.yml +++ b/.github/workflows/ci-test-limited-with-count.yml @@ -204,26 +204,19 @@ jobs: echo "specs_to_run=$specs_to_run" >> $GITHUB_ENV - # In case of run-id provided download the artifact from the previous run - - name: Download Docker image artifact - if: inputs.previous-workflow-run-id != 0 - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{ inputs.previous-workflow-run-id }} - - # In case of run-id is 0 download the artifact from the current run - - name: Download Docker image artifact - if: inputs.previous-workflow-run-id == 0 - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{github.run_id}} - - - name: Load Docker image from tar file + # The build job pushed the SUT image to GHCR tagged with its run id; + # when previous-workflow-run-id is set, reuse the image built by that + # earlier run instead. + - name: Load Docker image run: | - gunzip cicontainer.tar.gz - docker load -i cicontainer.tar + image_run_id='${{ inputs.previous-workflow-run-id }}' + if [[ "$image_run_id" == '0' ]]; then + image_run_id='${{ github.run_id }}' + fi + ghcr_image="ghcr.io/${{ github.repository }}-citest:run-$image_run_id" + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$ghcr_image" + docker tag "$ghcr_image" cicontainer - name: Create folder if: steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/ci-test-limited.yml b/.github/workflows/ci-test-limited.yml index c965f556841..c0d6672f695 100644 --- a/.github/workflows/ci-test-limited.yml +++ b/.github/workflows/ci-test-limited.yml @@ -115,26 +115,19 @@ jobs: specs_to_run=${specs_to_run#,} echo "specs_to_run=$specs_to_run" >> $GITHUB_ENV - # In case of run-id provided download the artifact from the previous run - - name: Download Docker image artifact - if: inputs.previous-workflow-run-id != 0 - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{ inputs.previous-workflow-run-id }} - - # In case of run-id is 0 download the artifact from the current run - - name: Download Docker image artifact - if: inputs.previous-workflow-run-id == 0 - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{github.run_id}} - - - name: Load Docker image from tar file + # The build job pushed the SUT image to GHCR tagged with its run id; + # when previous-workflow-run-id is set, reuse the image built by that + # earlier run instead. + - name: Load Docker image run: | - gunzip cicontainer.tar.gz - docker load -i cicontainer.tar + image_run_id='${{ inputs.previous-workflow-run-id }}' + if [[ "$image_run_id" == '0' ]]; then + image_run_id='${{ github.run_id }}' + fi + ghcr_image="ghcr.io/${{ github.repository }}-citest:run-$image_run_id" + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$ghcr_image" + docker tag "$ghcr_image" cicontainer - name: Create folder if: steps.run_result.outputs.run_result != 'success' diff --git a/.github/workflows/ci-test-playwright.yml b/.github/workflows/ci-test-playwright.yml index 3a63999e9dd..9348eb57953 100644 --- a/.github/workflows/ci-test-playwright.yml +++ b/.github/workflows/ci-test-playwright.yml @@ -91,27 +91,28 @@ jobs: password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Get Docker image id + env: + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} run: | - if [[ '${{ inputs.docker_image_name }}' != '' ]]; then - echo 'docker_container_name=${{ inputs.docker_image_name }}' >> "$GITHUB_ENV" + if [[ -n "$DOCKER_IMAGE_NAME" ]]; then + echo "docker_container_name=$DOCKER_IMAGE_NAME" >> "$GITHUB_ENV" else echo 'docker_container_name=cicontainer' >> "$GITHUB_ENV" fi - - name: Restore Docker image cache - if: inputs.docker_image_name == '' - uses: actions/cache@v4 - with: - path: cicontainer.tar.gz - key: docker-image-${{ github.run_id }} - + # The build job pushed the SUT image to GHCR tagged with this run's id; + # pull it from there (layer-parallel, no single-blob cache bottleneck). - name: Load Docker image + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }}-citest:run-${{ github.run_id }} + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} run: | - if [[ '${{ inputs.docker_image_name }}' == '' ]]; then - gunzip cicontainer.tar.gz - docker load -i cicontainer.tar + if [[ -z "$DOCKER_IMAGE_NAME" ]]; then + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker pull "$GHCR_IMAGE" + docker tag "$GHCR_IMAGE" cicontainer else - docker pull ${{ env.docker_container_name }} + docker pull "$DOCKER_IMAGE_NAME" fi - name: Create Appsmith stacks folder diff --git a/.github/workflows/ghcr-citest-retention.yml b/.github/workflows/ghcr-citest-retention.yml new file mode 100644 index 00000000000..9b5881736a0 --- /dev/null +++ b/.github/workflows/ghcr-citest-retention.yml @@ -0,0 +1,51 @@ +name: GHCR citest image retention + +# The build-docker-image workflow pushes a ~2 GB SUT image to GHCR tagged +# run- on every run. Test jobs only pull images from the current run, +# or (for /ci-test-limit reruns) from a recent previous run, so anything older +# than the retention window is dead weight. This prunes those versions daily. + +on: + schedule: + - cron: "47 3 * * *" + workflow_dispatch: + inputs: + older-than-days: + description: "Delete run-* versions older than this many days" + required: false + type: string + default: "7" + dry-run: + description: "Only log what would be deleted" + required: false + type: boolean + default: false + +permissions: + packages: write + +jobs: + prune: + runs-on: ubuntu-latest + steps: + - name: Delete citest package versions past the retention window + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OLDER_THAN_DAYS: ${{ inputs.older-than-days || '7' }} + DRY_RUN: ${{ inputs.dry-run && 'true' || 'false' }} + run: | + set -o errexit -o nounset -o pipefail + PACKAGE_PATH="/orgs/${GITHUB_REPOSITORY_OWNER}/packages/container/${GITHUB_REPOSITORY#*/}-citest" + cutoff=$(( $(date +%s) - OLDER_THAN_DAYS * 86400 )) + deleted=0 + while IFS=$'\t' read -r id created tags; do + echo "Deleting version $id (created $created, tags: $tags)" + if [[ "$DRY_RUN" != 'true' ]]; then + gh api --method DELETE "$PACKAGE_PATH/versions/$id" + fi + deleted=$((deleted + 1)) + done < <( + gh api --paginate "$PACKAGE_PATH/versions?per_page=100" \ + --jq ".[] | select((.created_at | fromdateiso8601) < $cutoff) | [.id, .created_at, (.metadata.container.tags | join(\",\"))] | @tsv" + ) + echo "Deleted $deleted version(s) older than $OLDER_THAN_DAYS day(s) (dry-run: $DRY_RUN)"