HelpersTask597_Configure_Codecov_for_coverage

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+# Glob pattern(s) of files to omit from the report.
+omit =
+    */devops/compose/*
+    */helpers/test/outcomes/*/tmp.scratch/*

.github/gh_requirements.txt

@@ -1,4 +1,5 @@
+boto3 >= 1.20.17
+coverage
 invoke
-tqdm
 s3fs
-boto3 >= 1.20.17
+tqdm

.github/workflows/coverage_tests.yml

@@ -0,0 +1,173 @@
+name: Test coverage
+
+on:
+  workflow_dispatch: {}
+  # every day at 00:00 UTC.
+  schedule:
+    - cron: '0 0 * * *'
+
+env:
+  CSFY_CI: true
+
+permissions:
+  # Required to authenticate and retrieve temporary AWS credentials via OIDC.
+  id-token: write
+  # Required to fetch and check out code from the repository.
+  contents: read
+  # Required to authenticate and pull Docker images from GitHub Container Registry (GHCR).
+  packages: read
+
+
+jobs:
+  run_test_coverage:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ vars.GH_ACTION_AWS_ROLE_ARN }}
+          role-session-name: ${{ vars.GH_ACTION_AWS_SESSION_NAME }}
+          aws-region: ${{ vars.CSFY_AWS_DEFAULT_REGION }}
+
+      - name: Login to GHCR
+        run: docker login ghcr.io -u gpsaggese -p ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cleanup
+        run: sudo chmod 777 -R .
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Update PYTHONPATH
+        run: echo "PYTHONPATH=.:helpers" >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r .github/gh_requirements.txt
+
+      - name: Pull image from GHCR
+        run: docker pull ghcr.io/${{ github.repository }}:dev
+
+      # Only on scheduled runs, capture ISO weekday (1=Mon … 7=Sun).
+      - name: Set DAY_OF_WEEK
+        if: github.event_name == 'schedule'
+        run: echo "DAY_OF_WEEK=$(date -u +'%u')" >> $GITHUB_ENV
+
+      # This step is used to trigger the fast test coverage generation using the invoke task.
+      - name: Run Fast test and generate report
+        id: run_fast
+        continue-on-error: true
+        run: |
+          echo "Simulating fast test failure"
+          exit 1
+        # env:
+        #   GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }}
+        #   CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        #   CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        #   CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
+        #   CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }}
+        #   CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }}
+        #   CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }}
+        # run: invoke run_coverage --suite fast
+
+      - name: Upload Fast Test Coverage to Codecov
+        id: upload_fast
+        # Only upload if the previous fast test run step succeeded (i.r report generated).
+        # failed step don’t generate a coverage report, so there's nothing to upload.
+        if: steps.run_fast.outcome == 'success'
+        continue-on-error: true
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          # Specify the Codecov flag name associated with this test suite.
+          # Required to separate coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI.
+          flags: fast
+          name: fast-test-coverage
+
+      - name: Run Slow test and generate report
+        id: run_slow
+        continue-on-error: true
+        env:
+          GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }}
+          CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+          CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
+          CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }}
+          CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }}
+          CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }}
+        run: invoke run_coverage --suite slow
+
+      - name: Upload Slow Test Coverage to Codecov
+        id: upload_slow
+        # Only upload if the previous slow test run step succeeded (i.r report generated).
+        if: steps.run_slow.outcome == 'success'
+        continue-on-error: true
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          flags: slow
+          name: slow-test-coverage
+
+      - name: Run Superslow test and generate report
+        # Run only on scheduled jobs or manual trigger
+        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+        env:
+          GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }}
+          CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+          CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
+          CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }}
+          CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }}
+          CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }}
+        run: |
+          # Only run superslow tests if it's Monday or a manual dispatch.
+          day_of_week=$(date +%u)
+          if [ "$day_of_week" = "1" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            invoke run_coverage --suite superslow
+          else
+            echo "Skipping superslow tests — not Monday and not manually triggered"
+            exit 0
+          fi
+
+      - name: Upload Superslow Test Coverage to Codecov
+        #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one.
+        if: steps.run_superslow.outcome == 'success'
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          flags: superslow
+          name: superslow-test-coverage
+
+        # Fail the job in CI if any of the fast/ slow run/ upload steps above failed.
+      - name: Fail if fast/slow test or upload failed
+        run: |
+          failed=""
+          if [ "${{ steps.run_fast.outcome }}" != "success" ]; then
+            echo "Fast test run failed"
+            failed="true"
+          fi
+          if [ "${{ steps.upload_fast.outcome }}" != "success" ]; then
+            echo "Fast test coverage upload failed"
+            failed="true"
+          fi
+          if [ "${{ steps.run_slow.outcome }}" != "success" ]; then
+            echo "Slow test run failed"
+            failed="true"
+          fi
+          if [ "${{ steps.upload_slow.outcome }}" != "success" ]; then
+            echo "Slow test coverage upload failed"
+            failed="true"
+          fi
+          if [ "$failed" = "true" ]; then
+            echo "At least one fast/slow test or upload step failed."
+            exit 1
+          fi