From eea3a1ce193c1d38091b71cb4d4d63aeb7005b32 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 21 Apr 2025 22:50:01 +0000 Subject: [PATCH 01/56] testing --- .coveragerc | 9 ++ .github/gh_requirements.txt | 5 +- .github/workflows/coverage_tests.yml | 119 +++++++++++++++ helpers/lib_tasks_pytest.py | 209 +++++++++++++++++++++++++++ tasks.py | 2 + 5 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 .coveragerc create mode 100644 .github/workflows/coverage_tests.yml diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..6a40fe195 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,9 @@ +[run] +# Tell coverage which directories should be considered as source. +source = helpers + +[report] +# Glob pattern(s) of files to omit from the report. +omit = + */devops/compose/* + */helpers/test/outcomes/*/tmp.scratch/* diff --git a/.github/gh_requirements.txt b/.github/gh_requirements.txt index b6eb64ad6..6a8e6997b 100644 --- a/.github/gh_requirements.txt +++ b/.github/gh_requirements.txt @@ -1,4 +1,5 @@ +boto3 >= 1.20.17 +coverage invoke -tqdm s3fs -boto3 >= 1.20.17 +tqdm diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml new file mode 100644 index 000000000..dd3593212 --- /dev/null +++ b/.github/workflows/coverage_tests.yml @@ -0,0 +1,119 @@ +name: Test coverage + +on: + # Run on every push (all branches). + push: {} + # Run on every pull request. + pull_request: {} + # Manual trigger + workflow_dispatch: {} + # Weekly cron (always runs on master under the hood) + schedule: + - cron: '0 0 * * 1' # every Monday at 00:00 UTC + +env: + CSFY_CI: true + +permissions: + id-token: write + contents: read + packages: read + +jobs: + run_test_coverage: + runs-on: ubuntu-latest + + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ vars.GH_ACTION_AWS_ROLE_ARN }} + role-session-name: ${{ vars.GH_ACTION_AWS_SESSION_NAME }} + aws-region: ${{ vars.CSFY_AWS_DEFAULT_REGION }} + + - name: Login to GHCR + run: docker login ghcr.io -u gpsaggese -p ${{ secrets.GITHUB_TOKEN }} + + - name: Cleanup + run: sudo chmod 777 -R . + + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update PYTHONPATH + run: echo "PYTHONPATH=.:helpers" >> $GITHUB_ENV + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r .github/gh_requirements.txt + + - name: Pull image from GHCR + run: docker pull ghcr.io/${{ github.repository }}:dev + + # This step is used to trigger the fast test coverage generation using the invoke task. + - name: Run Fast test and generate report + env: + GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} + CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} + CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} + run: invoke run_fast_coverage . + + # After the test run, rename the generated coverage.xml to coverage_fast.xml. + # - name: Rename Fast Test Coverage report + # if: always() + # run: mv ./coverage.xml ./coverage_fast.xml + + # This step is used to upload the fast test coverage report file to Codecov. + - name: Upload Fast Test Coverage to Codecov + # `if: always()` ensures this step runs regardless of prior failures. + # It is used here to always attempt uploading the coverage report, + # even if earlier steps have failed. + if: always() + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage_fast.xml + flags: fast + name: fast-test-coverage + # `fail_ci_if_error: true` instructs the action to mark the CI as failed + # if there is any error during the upload to Codecov. + fail_ci_if_error: true + + # This step is used to trigger the slow test coverage generation using the invoke task. + - name: Run Slow test and generate report + env: + GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} + CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} + CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} + run: invoke run_slow_coverage . + + # # Rename the slow test coverage report to avoid conflict with the fast tests coverage file. + # - name: Rename Slow Test Coverage report + # if: always() + # run: mv ./coverage.xml ./coverage_slow.xml + + - name: Upload Slow Test Coverage to Codecov + # "if: always()" ensures this step is executed regardless of previous outcomes. + if: always() + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage_slow.xml + flags: slow + name: slow-test-coverage + # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails, + # helping to catch issues with the coverage reporting. + fail_ci_if_error: true diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 3ad73449a..9300f406c 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -874,6 +874,215 @@ def run_coverage_report( # type: ignore _publish_html_coverage_report_on_s3(aws_profile) +def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: + """ + Helper function to determine the directories to include and exclude in the + coverage report based on the target directory. + + :param target_dir: directory for coverage stats. Use "." to indicate all directories + :return: tuple (include_in_report, exclude_from_report) where: + - include_in_report is a glob pattern for files to include + - exclude_from_report is a comma-separated glob pattern for files to exclude (or None if not applicable) + """ + if target_dir == ".": + include_in_report: str = "*" + exclude_from_report: Optional[str] = None + if hserver.skip_submodules_test(): + submodule_paths: List[str] = hgit.get_submodule_paths() + exclude_from_report = ",".join( + f"{path}/*" for path in submodule_paths + ) + else: + include_in_report = f"*/{target_dir}/*" + exclude_from_report = None + return include_in_report, exclude_from_report + + +@task +def run_fast_coverage( + ctx, target_dir: str, generate_html_report: bool = True +) -> str: + """ + Run fast tests with coverage and generate an XML report for Codecov. + + The task: + - Runs fast tests with coverage enabled. + - Collects coverage data and renames the resulting coverage file to .coverage_fast_tests. + - Combines any generated parallel coverage files. + - Outputs text, HTML (if requested), and XML reports. + + :param ctx: Invoke context. + :param target_dir: Directory to compute coverage stats for. + Use "." for all directories in the current working directory. + :param generate_html_report: Flag to indicate whether to generate an HTML coverage report. + :return: The filename of the fast tests coverage file. + """ + report_cmd: List[str] = [] + # Run fast tests and collect coverage data. + fast_tests_cmd: str = f"invoke run_fast_tests --coverage -p {target_dir}" + hlitauti.run(ctx, fast_tests_cmd, use_system=False) + fast_tests_coverage_file: str = ".coverage_fast_tests" + create_fast_tests_file_cmd: str = f"mv .coverage {fast_tests_coverage_file}" + hsystem.system(create_fast_tests_file_cmd) + report_cmd.append("coverage erase") + # Get the include and exclude patterns for the coverage report. + include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) + # Combine the fast test coverage data. + report_cmd.append(f"coverage combine --keep {fast_tests_coverage_file}") + # Generate text report with the coverage stats. + report_stats_cmd: str = ( + f"coverage report --include={include_in_report} --sort=Cover" + ) + if exclude_from_report: + report_stats_cmd += f" --omit={exclude_from_report}" + report_cmd.append(report_stats_cmd) + if generate_html_report: + # Generate HTML report with the coverage stats. + report_html_cmd: str = f"coverage html --include={include_in_report}" + if exclude_from_report: + report_html_cmd += f" --omit={exclude_from_report}" + report_cmd.append(report_html_cmd) + # Generate an XML report for Codecov. + report_cmd.append("coverage xml -o coverage.xml") + # Execute the generated commands inside Docker. + full_report_cmd: str = " && ".join(report_cmd) + docker_cmd_: str = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" + hlitauti.run(ctx, docker_cmd_) + return fast_tests_coverage_file + + +@task +def run_slow_coverage( + ctx, target_dir: str, generate_html_report: bool = True +) -> str: + """ + Run slow tests with coverage and generate an XML report for Codecov. + + The task: + - Runs slow tests with coverage enabled in parallel mode + - Combines any generated parallel coverage files + - Outputs text, HTML (if requested), and XML reports + + :param ctx: Invoke context. + :param target_dir: Directory to compute coverage stats for + Use "." for all directories in the current working directory + :param generate_html_report: Flag to indicate whether to generate an HTML coverage report + :return: The filename of the slow tests coverage file + """ + report_cmd: List[str] = [] + # Run slow tests and collect coverage data. + slow_tests_cmd: str = f"invoke run_slow_tests --coverage -p {target_dir}" + hlitauti.run(ctx, slow_tests_cmd, use_system=False) + slow_tests_coverage_file: str = ".coverage_slow_tests" + create_slow_tests_file_cmd: str = f"mv .coverage {slow_tests_coverage_file}" + hsystem.system(create_slow_tests_file_cmd) + hdbg.dassert_file_exists(slow_tests_coverage_file) + report_cmd.append("coverage erase") + # Get the include and exclude patterns for the coverage report. + include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) + # Combine the slow test coverage data. + report_cmd.append(f"coverage combine --keep {slow_tests_coverage_file}") + # Generate text report with the coverage stats. + report_stats_cmd: str = ( + f"coverage report --include={include_in_report} --sort=Cover" + ) + if exclude_from_report: + report_stats_cmd += f" --omit={exclude_from_report}" + report_cmd.append(report_stats_cmd) + if generate_html_report: + # Generate HTML report with the coverage stats. + report_html_cmd: str = f"coverage html --include={include_in_report}" + if exclude_from_report: + report_html_cmd += f" --omit={exclude_from_report}" + report_cmd.append(report_html_cmd) + # Generate an XML report for Codecov. + report_cmd.append("coverage xml -o coverage.xml") + # Execute the generated commands inside Docker. + full_report_cmd: str = " && ".join(report_cmd) + docker_cmd_: str = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" + hlitauti.run(ctx, docker_cmd_) + return slow_tests_coverage_file + + +def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: + """ + Determine include/exclude patterns for coverage based on target_dir. + + :param target_dir: directory to cover ("." for all directories) + :return: (include_pattern, exclude_pattern or None) + """ + if target_dir == ".": + include_in_report = "*" + exclude_from_report: Optional[str] = None + if hserver.skip_submodules_test(): + submodule_paths: List[str] = hgit.get_submodule_paths() + exclude_from_report = ",".join( + f"{path}/*" for path in submodule_paths + ) + else: + include_in_report = f"*/{target_dir}/*" + exclude_from_report = None + return include_in_report, exclude_from_report + + +def _run_coverage_suite( + ctx, suite: str, target_dir: str, generate_html_report: bool = True +) -> str: + """ + Internal helper to run coverage for a given suite (fast/slow) and emit XML. + + :param suite: one of "fast" or "slow" + :param target_dir: coverage target directory + :param generate_html_report: whether to produce HTML output + :return: filename of the suite-specific coverage data file + """ + data_file = f".coverage_{suite}_tests" + test_cmd = f"invoke run_{suite}_tests --coverage -p {target_dir}" + hlitauti.run(ctx, test_cmd, use_system=False) + hsystem.system(f"mv .coverage {data_file}") + include, exclude = _get_inclusion_settings(target_dir) + xml_file = f"coverage_{suite}.xml" + commands: List[str] = [ + "coverage erase", + f"coverage report --include={include} --sort=Cover" + + (f" --omit={exclude}" if exclude else ""), + ] + if generate_html_report: + html_cmd = f"coverage html --include={include}" + ( + f" --omit={exclude}" if exclude else "" + ) + commands.append(html_cmd) + commands.append(f"coverage xml -o {xml_file}") + full_cmd = " && ".join(commands) + docker_cmd = f"invoke docker_cmd --use-bash --cmd '{full_cmd}'" + hlitauti.run(ctx, docker_cmd) + return xml_file + + +@task +def run_fast_coverage( + ctx, target_dir: str, generate_html_report: bool = True +) -> str: + """ + Run fast tests with coverage and produce XML for Codecov. + """ + return _run_coverage_suite( + ctx, "fast", target_dir, generate_html_report=False + ) + + +@task +def run_slow_coverage( + ctx, target_dir: str, generate_html_report: bool = True +) -> str: + """ + Run slow tests with coverage and produce XML for Codecov. + """ + return _run_coverage_suite( + ctx, "slow", target_dir, generate_html_report=False + ) + + # ############################################################################# # Traceback. # ############################################################################# diff --git a/tasks.py b/tasks.py index 7a0362e8b..037dec7d8 100644 --- a/tasks.py +++ b/tasks.py @@ -92,6 +92,8 @@ pytest_repro, run_blank_tests, run_coverage_report, + run_fast_coverage, + run_slow_coverage, run_fast_slow_superslow_tests, run_fast_slow_tests, run_fast_tests, From 5d295012828ab8786c3d6c00463972b9f6db8643 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 21 Apr 2025 23:01:03 +0000 Subject: [PATCH 02/56] testing --- .github/workflows/coverage_tests.yml | 16 +++++++++------- helpers/lib_tasks_pytest.py | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index dd3593212..6ea71dfa1 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,16 +1,18 @@ name: Test coverage on: - # Run on every push (all branches). - push: {} - # Run on every pull request. + # Only run push builds when master changes + push: + branches: + - master + + # Run on every PR (regardless of source branch) pull_request: {} - # Manual trigger + workflow_dispatch: {} - # Weekly cron (always runs on master under the hood) - schedule: - - cron: '0 0 * * 1' # every Monday at 00:00 UTC + schedule: + - cron: '0 0 * * 1' env: CSFY_CI: true diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 9300f406c..30fa389ad 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -1044,6 +1044,7 @@ def _run_coverage_suite( xml_file = f"coverage_{suite}.xml" commands: List[str] = [ "coverage erase", + f"coverage combine --keep {data_file}" f"coverage report --include={include} --sort=Cover" + (f" --omit={exclude}" if exclude else ""), ] From f6eb29d20a3d63d38ca9af3efe06a3219f24541f Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 21 Apr 2025 23:13:33 +0000 Subject: [PATCH 03/56] testing --- helpers/lib_tasks_pytest.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 30fa389ad..24dd817a4 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -1037,23 +1037,20 @@ def _run_coverage_suite( :return: filename of the suite-specific coverage data file """ data_file = f".coverage_{suite}_tests" + xml_file = f"coverage_{suite}.xml" test_cmd = f"invoke run_{suite}_tests --coverage -p {target_dir}" hlitauti.run(ctx, test_cmd, use_system=False) hsystem.system(f"mv .coverage {data_file}") include, exclude = _get_inclusion_settings(target_dir) - xml_file = f"coverage_{suite}.xml" - commands: List[str] = [ - "coverage erase", - f"coverage combine --keep {data_file}" - f"coverage report --include={include} --sort=Cover" - + (f" --omit={exclude}" if exclude else ""), - ] + commands: List[str] = [] if generate_html_report: - html_cmd = f"coverage html --include={include}" + ( - f" --omit={exclude}" if exclude else "" + html_cmd = ( + f"coverage html --data-file {data_file} --include={include}" + + (f" --omit={exclude}" if exclude else "") ) commands.append(html_cmd) - commands.append(f"coverage xml -o {xml_file}") + xml_cmd = f"coverage xml --data-file {data_file} --output {xml_file}" + commands.append(xml_cmd) full_cmd = " && ".join(commands) docker_cmd = f"invoke docker_cmd --use-bash --cmd '{full_cmd}'" hlitauti.run(ctx, docker_cmd) From d33dd79957f509a7889b23b1f01eb86d28647a95 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 21 Apr 2025 23:19:56 +0000 Subject: [PATCH 04/56] testing --- helpers/lib_tasks_pytest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 24dd817a4..6a0465845 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -1049,7 +1049,7 @@ def _run_coverage_suite( + (f" --omit={exclude}" if exclude else "") ) commands.append(html_cmd) - xml_cmd = f"coverage xml --data-file {data_file} --output {xml_file}" + xml_cmd = f"coverage xml -o coverage_fast.xml --data-file .coverage_fast_tests" commands.append(xml_cmd) full_cmd = " && ".join(commands) docker_cmd = f"invoke docker_cmd --use-bash --cmd '{full_cmd}'" From bd3cf41f5d7f859fc189b9bc13ef707a7d04ea6f Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 21 Apr 2025 23:30:35 +0000 Subject: [PATCH 05/56] testing --- helpers/lib_tasks_pytest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 6a0465845..864a91cca 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -1049,12 +1049,11 @@ def _run_coverage_suite( + (f" --omit={exclude}" if exclude else "") ) commands.append(html_cmd) - xml_cmd = f"coverage xml -o coverage_fast.xml --data-file .coverage_fast_tests" - commands.append(xml_cmd) + commands.append(f"cp {data_file} .coverage") + commands.append(f"coverage xml -o {xml_file}") full_cmd = " && ".join(commands) docker_cmd = f"invoke docker_cmd --use-bash --cmd '{full_cmd}'" hlitauti.run(ctx, docker_cmd) - return xml_file @task From fac66d60397add9af1c5503ba3bc97c0504ed0db Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 21 Apr 2025 23:48:42 +0000 Subject: [PATCH 06/56] testing --- .github/workflows/coverage_tests.yml | 4 +- helpers/lib_tasks_pytest.py | 84 ++++++++++++++++++---------- 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 6ea71dfa1..e50639938 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -83,7 +83,7 @@ jobs: uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage_fast.xml + files: ./coverage.xml flags: fast name: fast-test-coverage # `fail_ci_if_error: true` instructs the action to mark the CI as failed @@ -113,7 +113,7 @@ jobs: uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage_slow.xml + files: ./coverage.xml flags: slow name: slow-test-coverage # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails, diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 864a91cca..d1e218470 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -1006,13 +1006,15 @@ def run_slow_coverage( def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: """ - Determine include/exclude patterns for coverage based on target_dir. + Determine include/omit glob patterns for the coverage report. - :param target_dir: directory to cover ("." for all directories) - :return: (include_pattern, exclude_pattern or None) + :param target_dir: directory for coverage stats; use "." to indicate all directories + :return: tuple (include_in_report, exclude_from_report) where + - include_in_report is the glob pattern to include + - exclude_from_report is a comma‑separated glob pattern to omit, or ``None`` """ if target_dir == ".": - include_in_report = "*" + include_in_report: str = "*" exclude_from_report: Optional[str] = None if hserver.skip_submodules_test(): submodule_paths: List[str] = hgit.get_submodule_paths() @@ -1025,8 +1027,11 @@ def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: return include_in_report, exclude_from_report -def _run_coverage_suite( - ctx, suite: str, target_dir: str, generate_html_report: bool = True +def _run_coverage( + ctx, + suite: str, + target_dir: str, + generate_html_report: bool, ) -> str: """ Internal helper to run coverage for a given suite (fast/slow) and emit XML. @@ -1034,26 +1039,39 @@ def _run_coverage_suite( :param suite: one of "fast" or "slow" :param target_dir: coverage target directory :param generate_html_report: whether to produce HTML output - :return: filename of the suite-specific coverage data file + :return: filename of the suite‑specific coverage data file """ - data_file = f".coverage_{suite}_tests" - xml_file = f"coverage_{suite}.xml" + + if suite not in ("fast", "slow"): + raise ValueError( + f"Unsupported suite='{suite}'. Expected 'fast' or 'slow'." + ) test_cmd = f"invoke run_{suite}_tests --coverage -p {target_dir}" + coverage_file = f".coverage_{suite}_tests" hlitauti.run(ctx, test_cmd, use_system=False) - hsystem.system(f"mv .coverage {data_file}") - include, exclude = _get_inclusion_settings(target_dir) - commands: List[str] = [] + hsystem.system(f"mv .coverage {coverage_file}") + hdbg.dassert_file_exists(coverage_file) + include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) + report_cmd: List[str] = [ + "coverage erase", + f"coverage combine --keep {coverage_file}", + ] + report_stats_cmd: str = ( + f"coverage report --include={include_in_report} --sort=Cover" + ) + if exclude_from_report: + report_stats_cmd += f" --omit={exclude_from_report}" + report_cmd.append(report_stats_cmd) if generate_html_report: - html_cmd = ( - f"coverage html --data-file {data_file} --include={include}" - + (f" --omit={exclude}" if exclude else "") - ) - commands.append(html_cmd) - commands.append(f"cp {data_file} .coverage") - commands.append(f"coverage xml -o {xml_file}") - full_cmd = " && ".join(commands) - docker_cmd = f"invoke docker_cmd --use-bash --cmd '{full_cmd}'" - hlitauti.run(ctx, docker_cmd) + report_html_cmd: str = f"coverage html --include={include_in_report}" + if exclude_from_report: + report_html_cmd += f" --omit={exclude_from_report}" + report_cmd.append(report_html_cmd) + report_cmd.append("coverage xml -o coverage.xml") + full_report_cmd: str = " && ".join(report_cmd) + docker_cmd_: str = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" + hlitauti.run(ctx, docker_cmd_) + return coverage_file @task @@ -1061,11 +1079,14 @@ def run_fast_coverage( ctx, target_dir: str, generate_html_report: bool = True ) -> str: """ - Run fast tests with coverage and produce XML for Codecov. + Task wrapper to run *fast* test suite with coverage and emit reports. + + :param ctx: Invoke context + :param target_dir: coverage target directory + :param generate_html_report: whether to produce HTML output + :return: filename of the fast‑suite coverage data file """ - return _run_coverage_suite( - ctx, "fast", target_dir, generate_html_report=False - ) + return _run_coverage(ctx, "fast", target_dir, generate_html_report) @task @@ -1073,11 +1094,14 @@ def run_slow_coverage( ctx, target_dir: str, generate_html_report: bool = True ) -> str: """ - Run slow tests with coverage and produce XML for Codecov. + Task wrapper to run *slow* test suite with coverage and emit reports. + + :param ctx: Invoke context + :param target_dir: coverage target directory + :param generate_html_report: whether to produce HTML output + :return: filename of the slow‑suite coverage data file """ - return _run_coverage_suite( - ctx, "slow", target_dir, generate_html_report=False - ) + return _run_coverage(ctx, "slow", target_dir, generate_html_report) # ############################################################################# From 0c20303bd9747de5d02823e853daa1411429fd1b Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 22 Apr 2025 00:14:20 +0000 Subject: [PATCH 07/56] testing --- .github/workflows/coverage_tests.yml | 17 ++++ helpers/lib_tasks_pytest.py | 132 +-------------------------- 2 files changed, 18 insertions(+), 131 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index e50639938..922bec940 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -16,6 +16,15 @@ on: env: CSFY_CI: true +flag_management: + default_rules: + carryforward: false + individual_flags: + - name: fast + carryforward: true + - name: slow + carryforward: true + permissions: id-token: write contents: read @@ -119,3 +128,11 @@ jobs: # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails, # helping to catch issues with the coverage reporting. fail_ci_if_error: true + + - name: Upload JUnit results to Codecov + if: always() + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: reports/junit_fast.xml,reports/junit_slow.xml + fail_ci_if_error: true diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index d1e218470..f86ed7e36 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -874,136 +874,6 @@ def run_coverage_report( # type: ignore _publish_html_coverage_report_on_s3(aws_profile) -def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: - """ - Helper function to determine the directories to include and exclude in the - coverage report based on the target directory. - - :param target_dir: directory for coverage stats. Use "." to indicate all directories - :return: tuple (include_in_report, exclude_from_report) where: - - include_in_report is a glob pattern for files to include - - exclude_from_report is a comma-separated glob pattern for files to exclude (or None if not applicable) - """ - if target_dir == ".": - include_in_report: str = "*" - exclude_from_report: Optional[str] = None - if hserver.skip_submodules_test(): - submodule_paths: List[str] = hgit.get_submodule_paths() - exclude_from_report = ",".join( - f"{path}/*" for path in submodule_paths - ) - else: - include_in_report = f"*/{target_dir}/*" - exclude_from_report = None - return include_in_report, exclude_from_report - - -@task -def run_fast_coverage( - ctx, target_dir: str, generate_html_report: bool = True -) -> str: - """ - Run fast tests with coverage and generate an XML report for Codecov. - - The task: - - Runs fast tests with coverage enabled. - - Collects coverage data and renames the resulting coverage file to .coverage_fast_tests. - - Combines any generated parallel coverage files. - - Outputs text, HTML (if requested), and XML reports. - - :param ctx: Invoke context. - :param target_dir: Directory to compute coverage stats for. - Use "." for all directories in the current working directory. - :param generate_html_report: Flag to indicate whether to generate an HTML coverage report. - :return: The filename of the fast tests coverage file. - """ - report_cmd: List[str] = [] - # Run fast tests and collect coverage data. - fast_tests_cmd: str = f"invoke run_fast_tests --coverage -p {target_dir}" - hlitauti.run(ctx, fast_tests_cmd, use_system=False) - fast_tests_coverage_file: str = ".coverage_fast_tests" - create_fast_tests_file_cmd: str = f"mv .coverage {fast_tests_coverage_file}" - hsystem.system(create_fast_tests_file_cmd) - report_cmd.append("coverage erase") - # Get the include and exclude patterns for the coverage report. - include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) - # Combine the fast test coverage data. - report_cmd.append(f"coverage combine --keep {fast_tests_coverage_file}") - # Generate text report with the coverage stats. - report_stats_cmd: str = ( - f"coverage report --include={include_in_report} --sort=Cover" - ) - if exclude_from_report: - report_stats_cmd += f" --omit={exclude_from_report}" - report_cmd.append(report_stats_cmd) - if generate_html_report: - # Generate HTML report with the coverage stats. - report_html_cmd: str = f"coverage html --include={include_in_report}" - if exclude_from_report: - report_html_cmd += f" --omit={exclude_from_report}" - report_cmd.append(report_html_cmd) - # Generate an XML report for Codecov. - report_cmd.append("coverage xml -o coverage.xml") - # Execute the generated commands inside Docker. - full_report_cmd: str = " && ".join(report_cmd) - docker_cmd_: str = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" - hlitauti.run(ctx, docker_cmd_) - return fast_tests_coverage_file - - -@task -def run_slow_coverage( - ctx, target_dir: str, generate_html_report: bool = True -) -> str: - """ - Run slow tests with coverage and generate an XML report for Codecov. - - The task: - - Runs slow tests with coverage enabled in parallel mode - - Combines any generated parallel coverage files - - Outputs text, HTML (if requested), and XML reports - - :param ctx: Invoke context. - :param target_dir: Directory to compute coverage stats for - Use "." for all directories in the current working directory - :param generate_html_report: Flag to indicate whether to generate an HTML coverage report - :return: The filename of the slow tests coverage file - """ - report_cmd: List[str] = [] - # Run slow tests and collect coverage data. - slow_tests_cmd: str = f"invoke run_slow_tests --coverage -p {target_dir}" - hlitauti.run(ctx, slow_tests_cmd, use_system=False) - slow_tests_coverage_file: str = ".coverage_slow_tests" - create_slow_tests_file_cmd: str = f"mv .coverage {slow_tests_coverage_file}" - hsystem.system(create_slow_tests_file_cmd) - hdbg.dassert_file_exists(slow_tests_coverage_file) - report_cmd.append("coverage erase") - # Get the include and exclude patterns for the coverage report. - include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) - # Combine the slow test coverage data. - report_cmd.append(f"coverage combine --keep {slow_tests_coverage_file}") - # Generate text report with the coverage stats. - report_stats_cmd: str = ( - f"coverage report --include={include_in_report} --sort=Cover" - ) - if exclude_from_report: - report_stats_cmd += f" --omit={exclude_from_report}" - report_cmd.append(report_stats_cmd) - if generate_html_report: - # Generate HTML report with the coverage stats. - report_html_cmd: str = f"coverage html --include={include_in_report}" - if exclude_from_report: - report_html_cmd += f" --omit={exclude_from_report}" - report_cmd.append(report_html_cmd) - # Generate an XML report for Codecov. - report_cmd.append("coverage xml -o coverage.xml") - # Execute the generated commands inside Docker. - full_report_cmd: str = " && ".join(report_cmd) - docker_cmd_: str = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" - hlitauti.run(ctx, docker_cmd_) - return slow_tests_coverage_file - - def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: """ Determine include/omit glob patterns for the coverage report. @@ -1046,7 +916,7 @@ def _run_coverage( raise ValueError( f"Unsupported suite='{suite}'. Expected 'fast' or 'slow'." ) - test_cmd = f"invoke run_{suite}_tests --coverage -p {target_dir}" + test_cmd = f"invoke run_{suite}_tests --coverage -p {target_dir} -- --junitxml=reports/junit_{suite}.xml -o junit_family=legacy" coverage_file = f".coverage_{suite}_tests" hlitauti.run(ctx, test_cmd, use_system=False) hsystem.system(f"mv .coverage {coverage_file}") From a5a72bb71b79d0004c8304ea298d9f137eaa9cbc Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 22 Apr 2025 00:22:54 +0000 Subject: [PATCH 08/56] testing --- .github/codecov.yml | 8 ++++++++ .github/workflows/coverage_tests.yml | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 .github/codecov.yml diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 000000000..d83ffa50d --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,8 @@ +flag_management: + default_rules: + carryforward: false + individual_flags: + - name: fast + carryforward: true + - name: slow + carryforward: true diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 922bec940..2e5783576 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -16,15 +16,6 @@ on: env: CSFY_CI: true -flag_management: - default_rules: - carryforward: false - individual_flags: - - name: fast - carryforward: true - - name: slow - carryforward: true - permissions: id-token: write contents: read From 7b7e2f169778d98074d4d559370e4be8b59bc739 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 22 Apr 2025 12:37:09 +0000 Subject: [PATCH 09/56] testing --- .github/workflows/coverage_tests.yml | 24 ++---------------------- .github/codecov.yml => codecov.yml | 6 ++++++ 2 files changed, 8 insertions(+), 22 deletions(-) rename .github/codecov.yml => codecov.yml (70%) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 2e5783576..61cc7b443 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,12 +1,12 @@ name: Test coverage on: - # Only run push builds when master changes + # Only run push builds when master changes. push: branches: - master - # Run on every PR (regardless of source branch) + # Run on every PR (regardless of source branch). pull_request: {} workflow_dispatch: {} @@ -69,12 +69,6 @@ jobs: CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} run: invoke run_fast_coverage . - # After the test run, rename the generated coverage.xml to coverage_fast.xml. - # - name: Rename Fast Test Coverage report - # if: always() - # run: mv ./coverage.xml ./coverage_fast.xml - - # This step is used to upload the fast test coverage report file to Codecov. - name: Upload Fast Test Coverage to Codecov # `if: always()` ensures this step runs regardless of prior failures. # It is used here to always attempt uploading the coverage report, @@ -90,7 +84,6 @@ jobs: # if there is any error during the upload to Codecov. fail_ci_if_error: true - # This step is used to trigger the slow test coverage generation using the invoke task. - name: Run Slow test and generate report env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} @@ -102,11 +95,6 @@ jobs: CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} run: invoke run_slow_coverage . - # # Rename the slow test coverage report to avoid conflict with the fast tests coverage file. - # - name: Rename Slow Test Coverage report - # if: always() - # run: mv ./coverage.xml ./coverage_slow.xml - - name: Upload Slow Test Coverage to Codecov # "if: always()" ensures this step is executed regardless of previous outcomes. if: always() @@ -119,11 +107,3 @@ jobs: # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails, # helping to catch issues with the coverage reporting. fail_ci_if_error: true - - - name: Upload JUnit results to Codecov - if: always() - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: reports/junit_fast.xml,reports/junit_slow.xml - fail_ci_if_error: true diff --git a/.github/codecov.yml b/codecov.yml similarity index 70% rename from .github/codecov.yml rename to codecov.yml index d83ffa50d..dd495636c 100644 --- a/.github/codecov.yml +++ b/codecov.yml @@ -6,3 +6,9 @@ flag_management: carryforward: true - name: slow carryforward: true + +comment: false +coverage: + status: + project: off + patch: off From 07aaefdc057fe169ee0a3da1daa9861dd63bcb67 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 22 Apr 2025 13:55:54 +0000 Subject: [PATCH 10/56] testing --- .github/workflows/coverage_tests.yml | 32 +++++++++++++++++++++++----- codecov.yml | 19 +++++++++++++++-- helpers/lib_tasks_pytest.py | 19 +++++++++++++++-- tasks.py | 1 + 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 61cc7b443..ef31be653 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -3,14 +3,10 @@ name: Test coverage on: # Only run push builds when master changes. push: - branches: - - master - + branches: [master ] # Run on every PR (regardless of source branch). pull_request: {} - workflow_dispatch: {} - schedule: - cron: '0 0 * * 1' env: @@ -80,6 +76,8 @@ jobs: files: ./coverage.xml flags: fast name: fast-test-coverage + # Combined coverage for tests will be checked rather than individual. + fail_ci: false # `fail_ci_if_error: true` instructs the action to mark the CI as failed # if there is any error during the upload to Codecov. fail_ci_if_error: true @@ -104,6 +102,30 @@ jobs: files: ./coverage.xml flags: slow name: slow-test-coverage + fail_ci: false # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails, # helping to catch issues with the coverage reporting. fail_ci_if_error: true + + - name: Run Superslow test and generate report + if: github.event_name == 'schedule' + env: + GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} + CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} + CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} + run: invoke run_superslow_coverage . + + - name: Upload Superslow Test Coverage to Codecov + if: github.event_name == 'schedule' + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: superslow + name: superslow-test-coverage + fail_ci: ${{ github.event_name == 'schedule' }} + fail_ci_if_error: true diff --git a/codecov.yml b/codecov.yml index dd495636c..0d1706a0c 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,9 +6,24 @@ flag_management: carryforward: true - name: slow carryforward: true + - name: superslow +# no per-line comments. comment: false + coverage: status: - project: off - patch: off + project: + # post the project‑level GitHub Check. + enabled: true + target_branch: master + # measure % drop relative to baseline. + comparator: relative + # fail if drop ≥1%. + threshold: 1 + flags: + - fast + - slow + - superslow + patch: + enabled: false diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index f86ed7e36..a978a9190 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -956,7 +956,7 @@ def run_fast_coverage( :param generate_html_report: whether to produce HTML output :return: filename of the fast‑suite coverage data file """ - return _run_coverage(ctx, "fast", target_dir, generate_html_report) + return _run_coverage(ctx, "fast", target_dir, generate_html_report=False) @task @@ -971,7 +971,22 @@ def run_slow_coverage( :param generate_html_report: whether to produce HTML output :return: filename of the slow‑suite coverage data file """ - return _run_coverage(ctx, "slow", target_dir, generate_html_report) + return _run_coverage(ctx, "slow", target_dir, generate_html_report=False) + + +@task +def run_superslow_coverage( + ctx, target_dir: str, generate_html_report: bool = True +) -> str: + """ + Task wrapper to run *slow* test suite with coverage and emit reports. + + :param ctx: Invoke context + :param target_dir: coverage target directory + :param generate_html_report: whether to produce HTML output + :return: filename of the slow‑suite coverage data file + """ + return _run_coverage(ctx, "superslow", target_dir, generate_html_report=False) # ############################################################################# diff --git a/tasks.py b/tasks.py index 037dec7d8..3a6ad6664 100644 --- a/tasks.py +++ b/tasks.py @@ -94,6 +94,7 @@ run_coverage_report, run_fast_coverage, run_slow_coverage, + run_superslow_coverage, run_fast_slow_superslow_tests, run_fast_slow_tests, run_fast_tests, From c41a94cb8363f1cbcc4daa778831b009f2dbf75d Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 22 Apr 2025 16:59:14 +0000 Subject: [PATCH 11/56] checkpoint --- .github/workflows/coverage_tests.yml | 45 +++++++++++++++++++++++----- codecov.yml | 6 ++-- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index ef31be653..be4f8bbbd 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,14 +1,13 @@ name: Test coverage on: - # Only run push builds when master changes. push: - branches: [master ] - # Run on every PR (regardless of source branch). - pull_request: {} + branches: [ master ] workflow_dispatch: {} + # every day at 00:00 UTC. schedule: - - cron: '0 0 * * 1' + - cron: '0 0 * * *' + env: CSFY_CI: true @@ -53,6 +52,11 @@ jobs: - name: Pull image from GHCR run: docker pull ghcr.io/${{ github.repository }}:dev + # Only on scheduled runs, capture ISO weekday (1=Mon … 7=Sun) + - name: Set DAY_OF_WEEK + if: github.event_name == 'schedule' + run: echo "DAY_OF_WEEK=$(date -u +'%u')" >> $GITHUB_ENV + # This step is used to trigger the fast test coverage generation using the invoke task. - name: Run Fast test and generate report env: @@ -107,8 +111,33 @@ jobs: # helping to catch issues with the coverage reporting. fail_ci_if_error: true + - name: Run Combined tests and generate report + env: + GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} + CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} + CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} + run: invoke run_coverage_report . + + - name: Upload Combined Test Coverage to Codecov + # "if: always()" ensures this step is executed regardless of previous outcomes. + if: always() + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: combined_fast_slow + name: combined-test-coverage + fail_ci: true + # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails, + # helping to catch issues with the coverage reporting. + fail_ci_if_error: true + - name: Run Superslow test and generate report - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -120,12 +149,12 @@ jobs: run: invoke run_superslow_coverage . - name: Upload Superslow Test Coverage to Codecov - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml flags: superslow name: superslow-test-coverage - fail_ci: ${{ github.event_name == 'schedule' }} + fail_ci: true fail_ci_if_error: true diff --git a/codecov.yml b/codecov.yml index 0d1706a0c..af881fcaf 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,6 +6,8 @@ flag_management: carryforward: true - name: slow carryforward: true + - name: combined_fast_slow + carryforward: true - name: superslow # no per-line comments. @@ -22,8 +24,6 @@ coverage: # fail if drop ≥1%. threshold: 1 flags: - - fast - - slow - - superslow + - combined_fast_slow patch: enabled: false From 48f52e9e151adf405c37852539bef42d392953d6 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 22 Apr 2025 18:08:58 +0000 Subject: [PATCH 12/56] checkpoint --- .../Master_buildmeister_dashboard.ipynb | 17 +++++++++ .../Master_buildmeister_dashboard.py | 35 +++++++------------ 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb index 091e2595e..f4cd2e296 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb +++ b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb @@ -299,6 +299,23 @@ "source": [ "http://172.30.2.44/html_coverage/runner_master/" ] + }, + { + "cell_type": "markdown", + "id": "027d1b3d", + "metadata": {}, + "source": [ + "# Code Coverage Page - CodeCov" + ] + }, + { + "cell_type": "markdown", + "id": "6c9219e1", + "metadata": {}, + "source": [ + "- Helpers: https://app.codecov.io/gh/causify-ai/helpers\n", + "- Other Repos: https://app.codecov.io/gh/causify-ai/" + ] } ], "metadata": { diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py index c7940be98..d52d95251 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py +++ b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py @@ -3,33 +3,28 @@ # jupytext: # text_representation: # extension: .py -# format_name: percent -# format_version: '1.3' -# jupytext_version: 1.15.2 +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.16.7 # kernelspec: # display_name: Python 3 (ipykernel) # language: python # name: python3 # --- -# %% [markdown] # TODO(Grisha): does it belong to the `devops` dir? -# %% [markdown] # # Description -# %% [markdown] # The notebook reports the latest build status for multiple repos. -# %% [markdown] # # Imports -# %% # %load_ext autoreload # %autoreload 2 # %matplotlib inline -# %% +# + import logging from typing import Dict @@ -41,23 +36,21 @@ import helpers.hpandas as hpandas import helpers.hprint as hprint import helpers.lib_tasks_gh as hlitagh +# - -# %% hdbg.init_logger(verbosity=logging.INFO) _LOG = logging.getLogger(__name__) _LOG.info("%s", henv.get_system_signature()[0]) hprint.config_notebook() -# %% # Set the display options to print the full table. pd.set_option("display.max_colwidth", None) pd.set_option("display.max_columns", None) -# %% [markdown] # # Utils -# %% +# + def make_clickable(url: str) -> str: """ Wrapper to make the URL value clickable. @@ -88,10 +81,10 @@ def color_format(val: str, status_color_mapping: Dict[str, str]) -> str: return f"background-color: {color}" -# %% [markdown] +# - + # # GH workflows state -# %% repo_list = [ "cryptokaizen/cmamp", "cryptokaizen/orange", @@ -106,7 +99,6 @@ def color_format(val: str, status_color_mapping: Dict[str, str]) -> str: workflow_df["url"] = workflow_df["url"].apply(make_clickable) _LOG.info(hpandas.df_to_str(workflow_df, log_level=logging.INFO)) -# %% status_color_mapping = { "success": "green", "failure": "red", @@ -129,24 +121,23 @@ def color_format(val: str, status_color_mapping: Dict[str, str]) -> str: ) ) -# %% [markdown] # # Allure reports -# %% [markdown] # - fast tests: http://172.30.2.44/allure_reports/cmamp/fast/latest/index.html # - slow tests: http://172.30.2.44/allure_reports/cmamp/slow/latest/index.html # - superslow tests: http://172.30.2.44/allure_reports/cmamp/superslow/latest/index.html -# %% [markdown] # # Number of open pull requests -# %% for repo in repo_list: number_prs = len(hlitagh.gh_get_open_prs(repo)) _LOG.info("%s: %s", repo, number_prs) -# %% [markdown] # # Code coverage HTML-page -# %% [markdown] # http://172.30.2.44/html_coverage/runner_master/ + +# # Code Coverage Page - CodeCov + +# - Helpers: https://app.codecov.io/gh/causify-ai/helpers +# - Other Repos: https://app.codecov.io/gh/causify-ai/ From a3495b760c82e9662c18044b64997e06da55e504 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 15:40:34 +0000 Subject: [PATCH 13/56] checkpoint --- codecov.yml | 12 ++- .../Master_buildmeister_dashboard.ipynb | 5 +- .../Master_buildmeister_dashboard.py | 2 +- helpers/lib_tasks_pytest.py | 101 +++++++++++++----- 4 files changed, 86 insertions(+), 34 deletions(-) diff --git a/codecov.yml b/codecov.yml index af881fcaf..09cedcab2 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,27 +1,31 @@ flag_management: default_rules: + # Do not reuse last coverage if missing. carryforward: false individual_flags: - name: fast + # Reuse last coverage if missing for true. carryforward: true - name: slow carryforward: true - name: combined_fast_slow carryforward: true - name: superslow + carryforward: true + -# no per-line comments. +# No per-line comments. comment: false coverage: status: project: - # post the project‑level GitHub Check. + # Post the project‑level GitHub Check. enabled: true target_branch: master - # measure % drop relative to baseline. + # Measure % drop relative to baseline. comparator: relative - # fail if drop ≥1%. + # Fail if drop ≥1%. threshold: 1 flags: - combined_fast_slow diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb index f4cd2e296..a0d0169ce 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb +++ b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb @@ -76,7 +76,7 @@ "import helpers.henv as henv\n", "import helpers.hpandas as hpandas\n", "import helpers.hprint as hprint\n", - "import helpers.lib_tasks_gh as hlitagh" + "import helpers.lib_tasks_gh as hlitagh\n" ] }, { @@ -313,8 +313,7 @@ "id": "6c9219e1", "metadata": {}, "source": [ - "- Helpers: https://app.codecov.io/gh/causify-ai/helpers\n", - "- Other Repos: https://app.codecov.io/gh/causify-ai/" + "- Helpers: https://app.codecov.io/gh/causify-ai/helpers" ] } ], diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py index d52d95251..1bf5e8341 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py +++ b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py @@ -36,6 +36,7 @@ import helpers.hpandas as hpandas import helpers.hprint as hprint import helpers.lib_tasks_gh as hlitagh + # - hdbg.init_logger(verbosity=logging.INFO) @@ -140,4 +141,3 @@ def color_format(val: str, status_color_mapping: Dict[str, str]) -> str: # # Code Coverage Page - CodeCov # - Helpers: https://app.codecov.io/gh/causify-ai/helpers -# - Other Repos: https://app.codecov.io/gh/causify-ai/ diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index a978a9190..8c50d6d10 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -876,16 +876,46 @@ def run_coverage_report( # type: ignore def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: """ - Determine include/omit glob patterns for the coverage report. + Determine include/omit glob patterns for the coverage report for both text + and HTML coverage reports. :param target_dir: directory for coverage stats; use "." to indicate all directories :return: tuple (include_in_report, exclude_from_report) where - include_in_report is the glob pattern to include - exclude_from_report is a comma‑separated glob pattern to omit, or ``None`` + + Examples: + 1. Cover everything (no submodules to omit): + ``` + _get_inclusion_settings(".") + ``` + ("*", "") + + 2. Only cover code under a specific directory: + ``` + _get_inclusion_settings("helpers") + ``` + ("*/helpers/*", None) + + Usage in `_run_coverage`: + + # Entire repo coverage (e.g. your 'helpers' project root): + ``` + include, omit = _get_inclusion_settings(".") + ``` + # coverage report --include=* --sort=Cover + # coverage html --include=* [--omit=ext/foo/*,ext/bar/*] + + # Single-directory coverage: + ``` + include, omit = _get_inclusion_settings("helpers") + ``` + # coverage report --include=*/helpers/* --sort=Cover + # coverage html --include=*/helpers/* [--omit=...] """ if target_dir == ".": - include_in_report: str = "*" - exclude_from_report: Optional[str] = None + include_in_report = "*" + exclude_from_report = "" if hserver.skip_submodules_test(): submodule_paths: List[str] = hgit.get_submodule_paths() exclude_from_report = ",".join( @@ -904,44 +934,71 @@ def _run_coverage( generate_html_report: bool, ) -> str: """ - Internal helper to run coverage for a given suite (fast/slow) and emit XML. + Run coverage for a given suite (fast/slow/superslow). :param suite: one of "fast" or "slow" :param target_dir: coverage target directory :param generate_html_report: whether to produce HTML output - :return: filename of the suite‑specific coverage data file + :return: xml file named 'coverage.xml' """ - if suite not in ("fast", "slow"): - raise ValueError( - f"Unsupported suite='{suite}'. Expected 'fast' or 'slow'." - ) - test_cmd = f"invoke run_{suite}_tests --coverage -p {target_dir} -- --junitxml=reports/junit_{suite}.xml -o junit_family=legacy" + hdbg.dassert_in( + suite, + ("fast", "slow", "superslow"), + "Expected one of 'fast', 'slow', or 'superslow'.", + ) + # Build the command line. + test_cmd = [ + # Invoke the "_tests" task. + "invoke", + f"run_{suite}_tests", + # Enable coverage collection. + "--coverage", + # Specify which directory to test. + "--path", + target_dir, + # Pass through remaining args to pytest. + "--", + # Emit JUnit XML report for CI systems. + "--junitxml=reports/junit_{suite}.xml", + # Force legacy JUnit output format. + "-o", + "junit_family=legacy", + ] + # Name output coverage data file for this suite. coverage_file = f".coverage_{suite}_tests" + # Run the tests under coverage. hlitauti.run(ctx, test_cmd, use_system=False) + # Move the default ".coverage" file to our suite-specific filename. hsystem.system(f"mv .coverage {coverage_file}") hdbg.dassert_file_exists(coverage_file) + # Compute which files/dirs to include and omit in the report. include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) report_cmd: List[str] = [ + # Clear out any prior coverage data. "coverage erase", + # Combine this suite's data into the coverage database. f"coverage combine --keep {coverage_file}", ] + # Generate a text report, including only our target paths. report_stats_cmd: str = ( f"coverage report --include={include_in_report} --sort=Cover" ) if exclude_from_report: report_stats_cmd += f" --omit={exclude_from_report}" report_cmd.append(report_stats_cmd) + # Produce HTML output for interactive browsing. if generate_html_report: report_html_cmd: str = f"coverage html --include={include_in_report}" if exclude_from_report: report_html_cmd += f" --omit={exclude_from_report}" report_cmd.append(report_html_cmd) + # Export XML coverage report. report_cmd.append("coverage xml -o coverage.xml") full_report_cmd: str = " && ".join(report_cmd) - docker_cmd_: str = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" + docker_cmd_ = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" + # Execute the full coverage/Docker pipeline. hlitauti.run(ctx, docker_cmd_) - return coverage_file @task @@ -956,7 +1013,8 @@ def run_fast_coverage( :param generate_html_report: whether to produce HTML output :return: filename of the fast‑suite coverage data file """ - return _run_coverage(ctx, "fast", target_dir, generate_html_report=False) + test = "fast" + _run_coverage(ctx, test, target_dir, generate_html_report=False) @task @@ -971,7 +1029,8 @@ def run_slow_coverage( :param generate_html_report: whether to produce HTML output :return: filename of the slow‑suite coverage data file """ - return _run_coverage(ctx, "slow", target_dir, generate_html_report=False) + test = "slow" + _run_coverage(ctx, test, target_dir, generate_html_report=False) @task @@ -986,7 +1045,8 @@ def run_superslow_coverage( :param generate_html_report: whether to produce HTML output :return: filename of the slow‑suite coverage data file """ - return _run_coverage(ctx, "superslow", target_dir, generate_html_report=False) + test = "superslow" + _run_coverage(ctx, test, target_dir, generate_html_report=False) # ############################################################################# @@ -1113,10 +1173,6 @@ def pytest_repro( # type: ignore pytest helpers/test/test_cache.py::TestCachingOnS3_2 ``` - "files": files with the failed tests, e.g., - ``` - pytest helpers/test/test_cache.py - pytest helpers/test/test_lib_tasks.py - ``` :param file_name: the name of the file containing the pytest output file to parse :param show_stacktrace: whether to show the stacktrace of the failed tests - only if it is available in the pytest output file @@ -1266,10 +1322,6 @@ def pytest_rename_test(ctx, old_test_class_name, new_test_class_name): # type: Rename the test and move its golden outcome. E.g., to rename a test class and all the test methods: - ``` - > i pytest_rename_test TestCacheUpdateFunction1 \ - TestCacheUpdateFunction_new - ``` :param old_test_class_name: old class name :param new_test_class_name: new class name @@ -1500,9 +1552,6 @@ def pytest_buildmeister( # type: ignore Run the regression tests. - Run updating all the tests - ``` - > pytest_buildmeister --pytest-opts "--update_outcomes" - ``` :param docker_clean: remove all dead Docker instances :param opts: options to pass to the invoke (e.g., `--version 1.2.0` to test From 714698b146389fbda59f6da56f394fbd482a62a6 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 16:57:45 +0000 Subject: [PATCH 14/56] test with superslow --- .github/workflows/coverage_tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index be4f8bbbd..47437c3f5 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -3,6 +3,7 @@ name: Test coverage on: push: branches: [ master ] + pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: @@ -137,7 +138,7 @@ jobs: fail_ci_if_error: true - name: Run Superslow test and generate report - if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' + if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -149,7 +150,7 @@ jobs: run: invoke run_superslow_coverage . - name: Upload Superslow Test Coverage to Codecov - if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' + if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} From de3b70f8ac306d6b9d5eb250fbdcca066c4fef5f Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 17:30:34 +0000 Subject: [PATCH 15/56] test --- helpers/lib_tasks_pytest.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 8c50d6d10..713a01dd0 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -8,6 +8,7 @@ import logging import os import re +import shlex import sys from typing import Any, List, Optional, Tuple, cast @@ -790,7 +791,7 @@ def _publish_html_coverage_report_on_s3(aws_profile: str) -> None: def run_coverage_report( # type: ignore ctx, target_dir, - generate_html_report=True, + generate_html_report=False, publish_html_on_s3=True, aws_profile="ck", ): @@ -931,7 +932,7 @@ def _run_coverage( ctx, suite: str, target_dir: str, - generate_html_report: bool, + generate_html_report: bool = False, ) -> str: """ Run coverage for a given suite (fast/slow/superslow). @@ -948,7 +949,7 @@ def _run_coverage( "Expected one of 'fast', 'slow', or 'superslow'.", ) # Build the command line. - test_cmd = [ + test_cmd_parts = [ # Invoke the "_tests" task. "invoke", f"run_{suite}_tests", @@ -965,6 +966,8 @@ def _run_coverage( "-o", "junit_family=legacy", ] + # join and quote them into a single shell command + test_cmd = " ".join(shlex.quote(p) for p in test_cmd_parts) # Name output coverage data file for this suite. coverage_file = f".coverage_{suite}_tests" # Run the tests under coverage. @@ -1002,9 +1005,7 @@ def _run_coverage( @task -def run_fast_coverage( - ctx, target_dir: str, generate_html_report: bool = True -) -> str: +def run_fast_coverage(ctx, target_dir: str) -> None: """ Task wrapper to run *fast* test suite with coverage and emit reports. @@ -1018,9 +1019,7 @@ def run_fast_coverage( @task -def run_slow_coverage( - ctx, target_dir: str, generate_html_report: bool = True -) -> str: +def run_slow_coverage(ctx, target_dir: str) -> None: """ Task wrapper to run *slow* test suite with coverage and emit reports. @@ -1034,9 +1033,7 @@ def run_slow_coverage( @task -def run_superslow_coverage( - ctx, target_dir: str, generate_html_report: bool = True -) -> str: +def run_superslow_coverage(ctx, target_dir: str) -> None: """ Task wrapper to run *slow* test suite with coverage and emit reports. From ff2bb19b91e452d2eea03c51d0f69bf4b6a7fea4 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 17:42:23 +0000 Subject: [PATCH 16/56] test --- helpers/lib_tasks_pytest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 713a01dd0..fd0126286 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -961,7 +961,7 @@ def _run_coverage( # Pass through remaining args to pytest. "--", # Emit JUnit XML report for CI systems. - "--junitxml=reports/junit_{suite}.xml", + f"--junitxml=reports/junit_{suite}.xml", # Force legacy JUnit output format. "-o", "junit_family=legacy", From 927dbfc9c9cd7f684d2f177e800abf5dbfe29a38 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 17:46:42 +0000 Subject: [PATCH 17/56] test --- helpers/lib_tasks_pytest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index fd0126286..d5840118e 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -956,7 +956,7 @@ def _run_coverage( # Enable coverage collection. "--coverage", # Specify which directory to test. - "--path", + "--p", target_dir, # Pass through remaining args to pytest. "--", From bbecbf7717580f99261bc359185d60a85787376b Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 17:55:00 +0000 Subject: [PATCH 18/56] test --- helpers/lib_tasks_pytest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index d5840118e..ac35ee882 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -956,7 +956,7 @@ def _run_coverage( # Enable coverage collection. "--coverage", # Specify which directory to test. - "--p", + "-p", target_dir, # Pass through remaining args to pytest. "--", From 335366fa55da7fb7a9b76e267b738966aee21199 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 18:29:35 +0000 Subject: [PATCH 19/56] flag test --- .github/workflows/coverage_tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 47437c3f5..5017440a0 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -138,7 +138,8 @@ jobs: fail_ci_if_error: true - name: Run Superslow test and generate report - if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' + # if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' + if: github.ref == 'refs/heads/master' || github.event_name == 'pull_request' env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -150,7 +151,8 @@ jobs: run: invoke run_superslow_coverage . - name: Upload Superslow Test Coverage to Codecov - if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' + # if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' + if: github.ref == 'refs/heads/master' || github.event_name == 'pull_request' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} From 04b7e00ba264efd6dafa0f56f82f1a31f02a078a Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 19:04:30 +0000 Subject: [PATCH 20/56] flag test --- .github/workflows/coverage_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 5017440a0..297f5c02b 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -139,7 +139,6 @@ jobs: - name: Run Superslow test and generate report # if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' - if: github.ref == 'refs/heads/master' || github.event_name == 'pull_request' env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -152,7 +151,6 @@ jobs: - name: Upload Superslow Test Coverage to Codecov # if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' - if: github.ref == 'refs/heads/master' || github.event_name == 'pull_request' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} From 609b0d2ad1ef52b1d1397753f8df963acb4ab3c9 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 19:16:29 +0000 Subject: [PATCH 21/56] flag test --- .github/workflows/coverage_tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 297f5c02b..b1ebdefcd 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,9 +1,6 @@ name: Test coverage on: - push: - branches: [ master ] - pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: @@ -138,7 +135,7 @@ jobs: fail_ci_if_error: true - name: Run Superslow test and generate report - # if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' + if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -150,7 +147,7 @@ jobs: run: invoke run_superslow_coverage . - name: Upload Superslow Test Coverage to Codecov - # if: github.event_name == 'schedule' && env.DAY_OF_WEEK == '1' || github.event_name == 'pull_request' + if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} From e31fceec0be8cd9d9f44d2c1e681f1196eb53a2a Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 19:48:37 +0000 Subject: [PATCH 22/56] test --- helpers/lib_tasks_pytest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index ac35ee882..2b4f8ad1f 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -835,9 +835,7 @@ def run_coverage_report( # type: ignore # does not affect the coverage results was added as a workaround. report_cmd.append("coverage erase") # Merge stats for fast and slow tests into single dir. - report_cmd.append( - f"coverage combine --keep {fast_tests_coverage_file} {slow_tests_coverage_file}" - ) + report_cmd.append(f"coverage combine --keep") # Specify the dirs to include and exclude in the report. exclude_from_report = None if target_dir == ".": From 22a063f63ee5864635c9ea03c113e154fc21c713 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 19:50:48 +0000 Subject: [PATCH 23/56] test --- .github/workflows/coverage_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index b1ebdefcd..4eb1950c8 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,6 +1,7 @@ name: Test coverage on: + pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: From f56b038cbb4730712b9d8f7113ed17c85cb24145 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 20:32:51 +0000 Subject: [PATCH 24/56] test --- .github/workflows/coverage_tests.yml | 27 +-------------------------- codecov.yml | 2 -- helpers/lib_tasks_pytest.py | 4 +++- 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 4eb1950c8..440b71d78 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -51,7 +51,7 @@ jobs: - name: Pull image from GHCR run: docker pull ghcr.io/${{ github.repository }}:dev - # Only on scheduled runs, capture ISO weekday (1=Mon … 7=Sun) + # Only on scheduled runs, capture ISO weekday (1=Mon … 7=Sun). - name: Set DAY_OF_WEEK if: github.event_name == 'schedule' run: echo "DAY_OF_WEEK=$(date -u +'%u')" >> $GITHUB_ENV @@ -110,31 +110,6 @@ jobs: # helping to catch issues with the coverage reporting. fail_ci_if_error: true - - name: Run Combined tests and generate report - env: - GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} - CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} - CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} - CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} - CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} - CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage_report . - - - name: Upload Combined Test Coverage to Codecov - # "if: always()" ensures this step is executed regardless of previous outcomes. - if: always() - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml - flags: combined_fast_slow - name: combined-test-coverage - fail_ci: true - # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails, - # helping to catch issues with the coverage reporting. - fail_ci_if_error: true - - name: Run Superslow test and generate report if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' env: diff --git a/codecov.yml b/codecov.yml index 09cedcab2..6710b0003 100644 --- a/codecov.yml +++ b/codecov.yml @@ -8,8 +8,6 @@ flag_management: carryforward: true - name: slow carryforward: true - - name: combined_fast_slow - carryforward: true - name: superslow carryforward: true diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 2b4f8ad1f..ac35ee882 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -835,7 +835,9 @@ def run_coverage_report( # type: ignore # does not affect the coverage results was added as a workaround. report_cmd.append("coverage erase") # Merge stats for fast and slow tests into single dir. - report_cmd.append(f"coverage combine --keep") + report_cmd.append( + f"coverage combine --keep {fast_tests_coverage_file} {slow_tests_coverage_file}" + ) # Specify the dirs to include and exclude in the report. exclude_from_report = None if target_dir == ".": From e9bf42a5cc6cba499e77a91ee11587fe5b148c22 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 20:57:22 +0000 Subject: [PATCH 25/56] test --- .github/workflows/coverage_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 440b71d78..6db323534 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,7 +1,6 @@ name: Test coverage on: - pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: From 6dd275d34d5925bb93fbe3dfe430aa6a3bddd357 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 24 Apr 2025 21:08:05 +0000 Subject: [PATCH 26/56] checkpoint --- .github/workflows/coverage_tests.yml | 1 + codecov.yml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 6db323534..bad2e60c1 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -110,6 +110,7 @@ jobs: fail_ci_if_error: true - name: Run Superslow test and generate report + # Scheduled for Monday (i.e Day 1) or when manually triggered. if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} diff --git a/codecov.yml b/codecov.yml index 6710b0003..92bd64a4e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -26,6 +26,8 @@ coverage: # Fail if drop ≥1%. threshold: 1 flags: - - combined_fast_slow + - fast + - slow + - superslow patch: enabled: false From 0ff65c1049f717fc721df0765c69d5a5ce90f9ba Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 28 Apr 2025 03:13:56 +0000 Subject: [PATCH 27/56] test --- .github/workflows/coverage_tests.yml | 13 +++-- helpers/lib_tasks_pytest.py | 74 ++++++++-------------------- tasks.py | 4 +- 3 files changed, 29 insertions(+), 62 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index bad2e60c1..54282426e 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,6 +1,7 @@ name: Test coverage on: + pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: @@ -35,7 +36,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - submodules: true + submodules: recursive fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} @@ -65,7 +66,7 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_fast_coverage . + run: invoke run_coverage --suite fast . - name: Upload Fast Test Coverage to Codecov # `if: always()` ensures this step runs regardless of prior failures. @@ -93,7 +94,7 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_slow_coverage . + run: invoke run_coverage --suite slow . - name: Upload Slow Test Coverage to Codecov # "if: always()" ensures this step is executed regardless of previous outcomes. @@ -110,6 +111,7 @@ jobs: fail_ci_if_error: true - name: Run Superslow test and generate report + id: run_superslow # Scheduled for Monday (i.e Day 1) or when manually triggered. if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' env: @@ -120,10 +122,11 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_superslow_coverage . + run: invoke run_coverage --suite superslow . - name: Upload Superslow Test Coverage to Codecov - if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' + #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one. + if: steps.run_superslow.outcome == 'success' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index ac35ee882..87418e1f1 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -8,7 +8,6 @@ import logging import os import re -import shlex import sys from typing import Any, List, Optional, Tuple, cast @@ -937,12 +936,11 @@ def _run_coverage( """ Run coverage for a given suite (fast/slow/superslow). - :param suite: one of "fast" or "slow" + :param suite: one of "fast", "slow" or "superslow" :param target_dir: coverage target directory :param generate_html_report: whether to produce HTML output :return: xml file named 'coverage.xml' """ - hdbg.dassert_in( suite, ("fast", "slow", "superslow"), @@ -958,30 +956,18 @@ def _run_coverage( # Specify which directory to test. "-p", target_dir, - # Pass through remaining args to pytest. - "--", - # Emit JUnit XML report for CI systems. - f"--junitxml=reports/junit_{suite}.xml", - # Force legacy JUnit output format. - "-o", - "junit_family=legacy", ] # join and quote them into a single shell command - test_cmd = " ".join(shlex.quote(p) for p in test_cmd_parts) - # Name output coverage data file for this suite. - coverage_file = f".coverage_{suite}_tests" + test_cmd = hlitauti.to_multi_line_cmd(test_cmd_parts) # Run the tests under coverage. hlitauti.run(ctx, test_cmd, use_system=False) - # Move the default ".coverage" file to our suite-specific filename. - hsystem.system(f"mv .coverage {coverage_file}") - hdbg.dassert_file_exists(coverage_file) + # Check that '.coverage' was created. + hdbg.dassert_file_exists(".coverage") # Compute which files/dirs to include and omit in the report. include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) report_cmd: List[str] = [ # Clear out any prior coverage data. - "coverage erase", - # Combine this suite's data into the coverage database. - f"coverage combine --keep {coverage_file}", + "coverage erase" ] # Generate a text report, including only our target paths. report_stats_cmd: str = ( @@ -1005,45 +991,25 @@ def _run_coverage( @task -def run_fast_coverage(ctx, target_dir: str) -> None: - """ - Task wrapper to run *fast* test suite with coverage and emit reports. - - :param ctx: Invoke context - :param target_dir: coverage target directory - :param generate_html_report: whether to produce HTML output - :return: filename of the fast‑suite coverage data file - """ - test = "fast" - _run_coverage(ctx, test, target_dir, generate_html_report=False) - - -@task -def run_slow_coverage(ctx, target_dir: str) -> None: - """ - Task wrapper to run *slow* test suite with coverage and emit reports. - - :param ctx: Invoke context - :param target_dir: coverage target directory - :param generate_html_report: whether to produce HTML output - :return: filename of the slow‑suite coverage data file - """ - test = "slow" - _run_coverage(ctx, test, target_dir, generate_html_report=False) - - -@task -def run_superslow_coverage(ctx, target_dir: str) -> None: +def run_coverage( + ctx, + suite: str, + target_dir: str = ".", +): """ - Task wrapper to run *slow* test suite with coverage and emit reports. + Unified task to run coverage for any test suite. :param ctx: Invoke context - :param target_dir: coverage target directory - :param generate_html_report: whether to produce HTML output - :return: filename of the slow‑suite coverage data file + :param suite: Test suite to run ("fast", "slow", "superslow") + :param target_dir: Target directory to measure coverage for (default + ".") """ - test = "superslow" - _run_coverage(ctx, test, target_dir, generate_html_report=False) + hdbg.dassert_in( + suite, + ("fast", "slow", "superslow"), + "Expected one of 'fast', 'slow', or 'superslow'.", + ) + _run_coverage(ctx, suite, target_dir, generate_html_report=False) # ############################################################################# diff --git a/tasks.py b/tasks.py index 3a6ad6664..5a1832e1e 100644 --- a/tasks.py +++ b/tasks.py @@ -92,9 +92,7 @@ pytest_repro, run_blank_tests, run_coverage_report, - run_fast_coverage, - run_slow_coverage, - run_superslow_coverage, + run_coverage, run_fast_slow_superslow_tests, run_fast_slow_tests, run_fast_tests, From 67f4c9f9ff2b64a32875af56b9f9db9a626732c4 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 28 Apr 2025 03:18:48 +0000 Subject: [PATCH 28/56] test --- .github/workflows/coverage_tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 54282426e..0032a7433 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -66,7 +66,7 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage --suite fast . + run: invoke run_coverage --suite fast -- . - name: Upload Fast Test Coverage to Codecov # `if: always()` ensures this step runs regardless of prior failures. @@ -94,7 +94,7 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage --suite slow . + run: invoke run_coverage --suite slow -- . - name: Upload Slow Test Coverage to Codecov # "if: always()" ensures this step is executed regardless of previous outcomes. @@ -122,7 +122,7 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage --suite superslow . + run: invoke run_coverage --suite superslow -- . - name: Upload Superslow Test Coverage to Codecov #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one. From 988d1ad606153e53c3ea5bcfbbba1bf73735c024 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 28 Apr 2025 03:37:37 +0000 Subject: [PATCH 29/56] test --- .github/workflows/coverage_tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 0032a7433..4c4918a04 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -113,7 +113,8 @@ jobs: - name: Run Superslow test and generate report id: run_superslow # Scheduled for Monday (i.e Day 1) or when manually triggered. - if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' + # if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' + if: always() env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -126,7 +127,8 @@ jobs: - name: Upload Superslow Test Coverage to Codecov #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one. - if: steps.run_superslow.outcome == 'success' + # if: steps.run_superslow.outcome == 'success' + if: always() uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} From f532944a5b450b4e57da9341f1bd729d0bc54a40 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Mon, 28 Apr 2025 03:49:53 +0000 Subject: [PATCH 30/56] checkpoint --- .github/workflows/coverage_tests.yml | 7 ++----- helpers/lib_tasks_pytest.py | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 4c4918a04..3294ef514 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,7 +1,6 @@ name: Test coverage on: - pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: @@ -113,8 +112,7 @@ jobs: - name: Run Superslow test and generate report id: run_superslow # Scheduled for Monday (i.e Day 1) or when manually triggered. - # if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' - if: always() + if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -127,8 +125,7 @@ jobs: - name: Upload Superslow Test Coverage to Codecov #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one. - # if: steps.run_superslow.outcome == 'success' - if: always() + if: steps.run_superslow.outcome == 'success' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 87418e1f1..3ff7c9ada 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -954,8 +954,7 @@ def _run_coverage( # Enable coverage collection. "--coverage", # Specify which directory to test. - "-p", - target_dir, + "-p", target_dir, ] # join and quote them into a single shell command test_cmd = hlitauti.to_multi_line_cmd(test_cmd_parts) From 0bbaadf23883ee9e4557560c9b8e4c9578be938b Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 29 Apr 2025 15:42:57 +0000 Subject: [PATCH 31/56] test --- .coveragerc | 9 --------- .github/workflows/coverage_tests.yml | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 6a40fe195..000000000 --- a/.coveragerc +++ /dev/null @@ -1,9 +0,0 @@ -[run] -# Tell coverage which directories should be considered as source. -source = helpers - -[report] -# Glob pattern(s) of files to omit from the report. -omit = - */devops/compose/* - */helpers/test/outcomes/*/tmp.scratch/* diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 3294ef514..0032a7433 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,6 +1,7 @@ name: Test coverage on: + pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: From 7e82eb4a49f3768c868d2735115fda27a3262dde Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 29 Apr 2025 17:50:22 +0000 Subject: [PATCH 32/56] test --- .coveragerc | 5 +++++ .github/workflows/coverage_tests.yml | 21 ++++++++++++++++----- codecov.yml | 18 +++++++++--------- helpers/lib_tasks_pytest.py | 18 +++++++----------- 4 files changed, 37 insertions(+), 25 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..9d4fbe2a0 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[report] +# Glob pattern(s) of files to omit from the report. +omit = + */devops/compose/* + */helpers/test/outcomes/*/tmp.scratch/* diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 0032a7433..f7de26618 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -11,10 +11,14 @@ env: CSFY_CI: true permissions: + # Required to authenticate and retrieve temporary AWS credentials via OIDC. id-token: write + # Required to fetch and check out code from the repository. contents: read + # Required to authenticate and pull Docker images from GitHub Container Registry (GHCR). packages: read + jobs: run_test_coverage: runs-on: ubuntu-latest @@ -58,6 +62,8 @@ jobs: # This step is used to trigger the fast test coverage generation using the invoke task. - name: Run Fast test and generate report + id: run_fast + if: always() env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -66,17 +72,19 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage --suite fast -- . + run: invoke run_coverage --suite fast - name: Upload Fast Test Coverage to Codecov # `if: always()` ensures this step runs regardless of prior failures. # It is used here to always attempt uploading the coverage report, # even if earlier steps have failed. - if: always() + if: steps.run_fast.outcome == 'success' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml + # Specify the Codecov flag name associated with this test suite. + # Required to separatw coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI. flags: fast name: fast-test-coverage # Combined coverage for tests will be checked rather than individual. @@ -86,6 +94,8 @@ jobs: fail_ci_if_error: true - name: Run Slow test and generate report + id: run_slow + continue-on-error: true env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -94,11 +104,11 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage --suite slow -- . + run: invoke run_coverage --suite slow - name: Upload Slow Test Coverage to Codecov # "if: always()" ensures this step is executed regardless of previous outcomes. - if: always() + if: steps.run_slow.outcome == 'success' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -112,6 +122,7 @@ jobs: - name: Run Superslow test and generate report id: run_superslow + continue-on-error: true # Scheduled for Monday (i.e Day 1) or when manually triggered. if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' env: @@ -122,7 +133,7 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage --suite superslow -- . + run: invoke run_coverage --suite superslow - name: Upload Superslow Test Coverage to Codecov #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one. diff --git a/codecov.yml b/codecov.yml index 92bd64a4e..0cb65c7db 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,10 +1,6 @@ flag_management: - default_rules: - # Do not reuse last coverage if missing. - carryforward: false individual_flags: - name: fast - # Reuse last coverage if missing for true. carryforward: true - name: slow carryforward: true @@ -12,22 +8,26 @@ flag_management: carryforward: true -# No per-line comments. + +# Disable per-line comments on pull requests. comment: false coverage: status: project: - # Post the project‑level GitHub Check. + # Post an overall project-level GitHub check summarizing coverage status. enabled: true + # Use the master branch as the baseline for comparison. target_branch: master - # Measure % drop relative to baseline. + # Compare coverage change relative to the baseline branch. comparator: relative - # Fail if drop ≥1%. + # Fail the status check if coverage drops by 1% or more. threshold: 1 + # Specify which flags to include in the project-level coverage calculation. flags: - fast - slow - superslow patch: - enabled: false + # Do not post patch-level coverage checks for pull requests. + enabled: true diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 3ff7c9ada..205630050 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -904,7 +904,7 @@ def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: include, omit = _get_inclusion_settings(".") ``` # coverage report --include=* --sort=Cover - # coverage html --include=* [--omit=ext/foo/*,ext/bar/*] + # coverage html --include=* [--omit=submodule1/*,submodule2/*] # Single-directory coverage: ``` @@ -954,7 +954,8 @@ def _run_coverage( # Enable coverage collection. "--coverage", # Specify which directory to test. - "-p", target_dir, + "-p", + target_dir, ] # join and quote them into a single shell command test_cmd = hlitauti.to_multi_line_cmd(test_cmd_parts) @@ -998,16 +999,11 @@ def run_coverage( """ Unified task to run coverage for any test suite. - :param ctx: Invoke context - :param suite: Test suite to run ("fast", "slow", "superslow") - :param target_dir: Target directory to measure coverage for (default - ".") + :param ctx: invoke context + :param suite: suite to run ("fast", "slow", "superslow") + :param target_dir: directory to measure coverage """ - hdbg.dassert_in( - suite, - ("fast", "slow", "superslow"), - "Expected one of 'fast', 'slow', or 'superslow'.", - ) + hdbg.dassert_in(suite, ("fast", "slow", "superslow")) _run_coverage(ctx, suite, target_dir, generate_html_report=False) From 6c6b587678e3e3f1b5cd32174d7d9bb5540af034 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 29 Apr 2025 18:03:26 +0000 Subject: [PATCH 33/56] test --- helpers/lib_tasks_pytest.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 205630050..3c4de28b6 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -951,7 +951,7 @@ def _run_coverage( # Invoke the "_tests" task. "invoke", f"run_{suite}_tests", - # Enable coverage collection. + # Enable coverage collection to track code execution. "--coverage", # Specify which directory to test. "-p", @@ -961,20 +961,16 @@ def _run_coverage( test_cmd = hlitauti.to_multi_line_cmd(test_cmd_parts) # Run the tests under coverage. hlitauti.run(ctx, test_cmd, use_system=False) - # Check that '.coverage' was created. hdbg.dassert_file_exists(".coverage") # Compute which files/dirs to include and omit in the report. include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) - report_cmd: List[str] = [ - # Clear out any prior coverage data. - "coverage erase" - ] # Generate a text report, including only our target paths. report_stats_cmd: str = ( f"coverage report --include={include_in_report} --sort=Cover" ) if exclude_from_report: report_stats_cmd += f" --omit={exclude_from_report}" + report_cmd: List[str] report_cmd.append(report_stats_cmd) # Produce HTML output for interactive browsing. if generate_html_report: @@ -982,7 +978,7 @@ def _run_coverage( if exclude_from_report: report_html_cmd += f" --omit={exclude_from_report}" report_cmd.append(report_html_cmd) - # Export XML coverage report. + # Export XML coverage report for Codecov integration. report_cmd.append("coverage xml -o coverage.xml") full_report_cmd: str = " && ".join(report_cmd) docker_cmd_ = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" From bac6d0be65c9a9de3be81a9fd6965232a7a6c11f Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 29 Apr 2025 18:14:27 +0000 Subject: [PATCH 34/56] test --- helpers/lib_tasks_pytest.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 3c4de28b6..42d8f120a 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -880,9 +880,7 @@ def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: and HTML coverage reports. :param target_dir: directory for coverage stats; use "." to indicate all directories - :return: tuple (include_in_report, exclude_from_report) where - - include_in_report is the glob pattern to include - - exclude_from_report is a comma‑separated glob pattern to omit, or ``None`` + :return: glob pattern to include and a comma-separated glob pattern to omit Examples: 1. Cover everything (no submodules to omit): @@ -951,7 +949,7 @@ def _run_coverage( # Invoke the "_tests" task. "invoke", f"run_{suite}_tests", - # Enable coverage collection to track code execution. + # enable coverage computation. "--coverage", # Specify which directory to test. "-p", @@ -970,7 +968,7 @@ def _run_coverage( ) if exclude_from_report: report_stats_cmd += f" --omit={exclude_from_report}" - report_cmd: List[str] + report_cmd: List[str] = [] report_cmd.append(report_stats_cmd) # Produce HTML output for interactive browsing. if generate_html_report: @@ -978,7 +976,7 @@ def _run_coverage( if exclude_from_report: report_html_cmd += f" --omit={exclude_from_report}" report_cmd.append(report_html_cmd) - # Export XML coverage report for Codecov integration. + # Export XML coverage report . report_cmd.append("coverage xml -o coverage.xml") full_report_cmd: str = " && ".join(report_cmd) docker_cmd_ = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" From 374b8af55dc7ab98be106ac403abf2a7f1403bf0 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 29 Apr 2025 18:31:55 +0000 Subject: [PATCH 35/56] test --- .github/workflows/coverage_tests.yml | 4 +++- helpers/lib_tasks_pytest.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index f7de26618..2fbb2b953 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -63,7 +63,9 @@ jobs: # This step is used to trigger the fast test coverage generation using the invoke task. - name: Run Fast test and generate report id: run_fast + # `if: always()` ensures this step runs regardless of prior failures. if: always() + continue-on-error: true env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -75,7 +77,6 @@ jobs: run: invoke run_coverage --suite fast - name: Upload Fast Test Coverage to Codecov - # `if: always()` ensures this step runs regardless of prior failures. # It is used here to always attempt uploading the coverage report, # even if earlier steps have failed. if: steps.run_fast.outcome == 'success' @@ -95,6 +96,7 @@ jobs: - name: Run Slow test and generate report id: run_slow + if: always() continue-on-error: true env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 42d8f120a..1ec45c70a 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -962,13 +962,16 @@ def _run_coverage( hdbg.dassert_file_exists(".coverage") # Compute which files/dirs to include and omit in the report. include_in_report, exclude_from_report = _get_inclusion_settings(target_dir) + report_cmd: List[str] = [ + # Reset any previous coverage data to avoid contamination. + "coverage erase" + ] # Generate a text report, including only our target paths. report_stats_cmd: str = ( f"coverage report --include={include_in_report} --sort=Cover" ) if exclude_from_report: report_stats_cmd += f" --omit={exclude_from_report}" - report_cmd: List[str] = [] report_cmd.append(report_stats_cmd) # Produce HTML output for interactive browsing. if generate_html_report: From 4fea59d69e1d8cd92f6caa1ffbb9e7ea1b7914b5 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 29 Apr 2025 18:54:18 +0000 Subject: [PATCH 36/56] checkpoint --- .github/workflows/coverage_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 2fbb2b953..7cf1f516b 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,7 +1,6 @@ name: Test coverage on: - pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: From bbba10f758768daec87b0178959250b3501e4089 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Tue, 29 Apr 2025 19:01:51 +0000 Subject: [PATCH 37/56] checkpoint --- .github/workflows/coverage_tests.yml | 6 ++---- codecov.yml | 2 +- helpers/lib_tasks_pytest.py | 5 ++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 7cf1f516b..dc8685e1a 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -76,8 +76,7 @@ jobs: run: invoke run_coverage --suite fast - name: Upload Fast Test Coverage to Codecov - # It is used here to always attempt uploading the coverage report, - # even if earlier steps have failed. + # Upload report if the fast test run successfully. if: steps.run_fast.outcome == 'success' uses: codecov/codecov-action@v5 with: @@ -117,8 +116,7 @@ jobs: flags: slow name: slow-test-coverage fail_ci: false - # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails, - # helping to catch issues with the coverage reporting. + # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails. fail_ci_if_error: true - name: Run Superslow test and generate report diff --git a/codecov.yml b/codecov.yml index 0cb65c7db..b64b9773a 100644 --- a/codecov.yml +++ b/codecov.yml @@ -29,5 +29,5 @@ coverage: - slow - superslow patch: - # Do not post patch-level coverage checks for pull requests. + # Post patch-level coverage checks for pull requests. enabled: true diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 1ec45c70a..d9a54f226 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -949,13 +949,12 @@ def _run_coverage( # Invoke the "_tests" task. "invoke", f"run_{suite}_tests", - # enable coverage computation. + # Enable coverage computation. "--coverage", # Specify which directory to test. "-p", target_dir, ] - # join and quote them into a single shell command test_cmd = hlitauti.to_multi_line_cmd(test_cmd_parts) # Run the tests under coverage. hlitauti.run(ctx, test_cmd, use_system=False) @@ -979,7 +978,7 @@ def _run_coverage( if exclude_from_report: report_html_cmd += f" --omit={exclude_from_report}" report_cmd.append(report_html_cmd) - # Export XML coverage report . + # Export XML coverage report to integrate with Codecov. report_cmd.append("coverage xml -o coverage.xml") full_report_cmd: str = " && ".join(report_cmd) docker_cmd_ = f"invoke docker_cmd --use-bash --cmd '{full_report_cmd}'" From f8240846aca1f9d54384e4f479cd34a82b31cc41 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 18:50:13 +0000 Subject: [PATCH 38/56] test --- .github/workflows/coverage_tests.yml | 32 ++++++++++++++-------------- codecov.yml | 4 +--- helpers/lib_tasks_pytest.py | 15 ++++--------- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index dc8685e1a..d4028557f 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,6 +1,7 @@ name: Test coverage on: + pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: @@ -62,8 +63,6 @@ jobs: # This step is used to trigger the fast test coverage generation using the invoke task. - name: Run Fast test and generate report id: run_fast - # `if: always()` ensures this step runs regardless of prior failures. - if: always() continue-on-error: true env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} @@ -76,8 +75,10 @@ jobs: run: invoke run_coverage --suite fast - name: Upload Fast Test Coverage to Codecov + id: upload_fast # Upload report if the fast test run successfully. if: steps.run_fast.outcome == 'success' + continue-on-error: true uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -86,15 +87,9 @@ jobs: # Required to separatw coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI. flags: fast name: fast-test-coverage - # Combined coverage for tests will be checked rather than individual. - fail_ci: false - # `fail_ci_if_error: true` instructs the action to mark the CI as failed - # if there is any error during the upload to Codecov. - fail_ci_if_error: true - name: Run Slow test and generate report id: run_slow - if: always() continue-on-error: true env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} @@ -107,21 +102,17 @@ jobs: run: invoke run_coverage --suite slow - name: Upload Slow Test Coverage to Codecov - # "if: always()" ensures this step is executed regardless of previous outcomes. + id: upload_slow if: steps.run_slow.outcome == 'success' + continue-on-error: true uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml flags: slow name: slow-test-coverage - fail_ci: false - # "fail_ci_if_error: true" causes the job to fail if the upload to Codecov fails. - fail_ci_if_error: true - name: Run Superslow test and generate report - id: run_superslow - continue-on-error: true # Scheduled for Monday (i.e Day 1) or when manually triggered. if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' env: @@ -143,5 +134,14 @@ jobs: files: ./coverage.xml flags: superslow name: superslow-test-coverage - fail_ci: true - fail_ci_if_error: true + + # Fail the job in CI if any of the run or upload steps above failed. + - name: Fail if fast/slow test or upload failed + run: | + if [ "${{ steps.run_fast.outcome }}" != "success" ] || \ + [ "${{ steps.run_slow.outcome }}" != "success" ] || \ + [ "${{ steps.upload_fast.outcome }}" != "success" ] || \ + [ "${{ steps.upload_slow.outcome }}" != "success" ]; then + echo "At least one fast/slow test or upload step failed." + exit 1 + fi diff --git a/codecov.yml b/codecov.yml index b64b9773a..6cce23813 100644 --- a/codecov.yml +++ b/codecov.yml @@ -7,8 +7,6 @@ flag_management: - name: superslow carryforward: true - - # Disable per-line comments on pull requests. comment: false @@ -29,5 +27,5 @@ coverage: - slow - superslow patch: - # Post patch-level coverage checks for pull requests. + # Enable a GitHub check that reports coverage for only the lines changed in the pull request. enabled: true diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index d9a54f226..9aaa51a12 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -930,20 +930,15 @@ def _run_coverage( suite: str, target_dir: str, generate_html_report: bool = False, -) -> str: +) -> None: """ Run coverage for a given suite (fast/slow/superslow). :param suite: one of "fast", "slow" or "superslow" :param target_dir: coverage target directory :param generate_html_report: whether to produce HTML output - :return: xml file named 'coverage.xml' """ - hdbg.dassert_in( - suite, - ("fast", "slow", "superslow"), - "Expected one of 'fast', 'slow', or 'superslow'.", - ) + hdbg.dassert_in(suite, ("fast", "slow", "superslow")) # Build the command line. test_cmd_parts = [ # Invoke the "_tests" task. @@ -988,9 +983,7 @@ def _run_coverage( @task def run_coverage( - ctx, - suite: str, - target_dir: str = ".", + ctx, suite: str, target_dir: str = ".", generate_html_report: bool = False ): """ Unified task to run coverage for any test suite. @@ -1000,7 +993,7 @@ def run_coverage( :param target_dir: directory to measure coverage """ hdbg.dassert_in(suite, ("fast", "slow", "superslow")) - _run_coverage(ctx, suite, target_dir, generate_html_report=False) + _run_coverage(ctx, suite, target_dir, generate_html_report) # ############################################################################# From 8992d8a8f29fbee842702096793781d60066aba9 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 19:03:31 +0000 Subject: [PATCH 39/56] test --- codecov.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codecov.yml b/codecov.yml index 6cce23813..373256262 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,4 +1,7 @@ flag_management: + default_rules: + # Delay GitHub status checks (e.g., patch/project) until all expected flags are uploaded. + wait_for_results: true individual_flags: - name: fast carryforward: true From 0b3e8e8f06cf1d9c9dae7f0ede48ababe5523de4 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 19:19:43 +0000 Subject: [PATCH 40/56] checkpoint --- .github/workflows/coverage_tests.yml | 7 +++---- helpers/lib_tasks_pytest.py | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index d4028557f..c77d470d8 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,7 +1,6 @@ name: Test coverage on: - pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: @@ -76,7 +75,7 @@ jobs: - name: Upload Fast Test Coverage to Codecov id: upload_fast - # Upload report if the fast test run successfully. + # Upload report if the fast test run successfully and report is generated. if: steps.run_fast.outcome == 'success' continue-on-error: true uses: codecov/codecov-action@v5 @@ -84,7 +83,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml # Specify the Codecov flag name associated with this test suite. - # Required to separatw coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI. + # Required to separate coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI. flags: fast name: fast-test-coverage @@ -135,7 +134,7 @@ jobs: flags: superslow name: superslow-test-coverage - # Fail the job in CI if any of the run or upload steps above failed. + # Fail the job in CI if any of the fast/ slow run/ upload steps above failed. - name: Fail if fast/slow test or upload failed run: | if [ "${{ steps.run_fast.outcome }}" != "success" ] || \ diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 9aaa51a12..32da831fb 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -947,8 +947,7 @@ def _run_coverage( # Enable coverage computation. "--coverage", # Specify which directory to test. - "-p", - target_dir, + "-p", target_dir, ] test_cmd = hlitauti.to_multi_line_cmd(test_cmd_parts) # Run the tests under coverage. From af9b90b70f4db5d31318b5db67e5923deb1e6382 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 20:04:13 +0000 Subject: [PATCH 41/56] test --- .github/workflows/coverage_tests.yml | 1 + codecov.yml | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index c77d470d8..5d43c6e80 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,6 +1,7 @@ name: Test coverage on: + pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: diff --git a/codecov.yml b/codecov.yml index 373256262..417c70605 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,8 +10,20 @@ flag_management: - name: superslow carryforward: true -# Disable per-line comments on pull requests. -comment: false +# Enable PR summary comment with coverage details. +comment: + # Define the layout of the summary comment to show overall, diff, and file-level stats. + layout: "reach, diff, files" + # Overwrite the previous comment instead of posting a new one each time. + behavior: default + # Allow comment to appear even if the coverage drop is small or unchanged. + require_changes: false + # Disable inline per-line critical path comments in the PR file view. + show_critical_paths: false + +# Disable GitHub inline annotations (e.g., red/yellow markers in 'Files changed'). +coverage: + annotations: false coverage: status: From 1e8850eb8a517c8a5fba4eb985f10d06d37c72f5 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 20:55:37 +0000 Subject: [PATCH 42/56] test --- codecov.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/codecov.yml b/codecov.yml index 417c70605..ab6b85fae 100644 --- a/codecov.yml +++ b/codecov.yml @@ -21,11 +21,10 @@ comment: # Disable inline per-line critical path comments in the PR file view. show_critical_paths: false -# Disable GitHub inline annotations (e.g., red/yellow markers in 'Files changed'). -coverage: - annotations: false - coverage: + # Disable GitHub inline annotations (e.g., red/yellow markers in 'Files changed'). + coverage: + annotations: false status: project: # Post an overall project-level GitHub check summarizing coverage status. From 0aba958de8d29797555d65fb4ed1a3e89067c7b3 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 21:15:00 +0000 Subject: [PATCH 43/56] test --- codecov.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codecov.yml b/codecov.yml index ab6b85fae..4054f3213 100644 --- a/codecov.yml +++ b/codecov.yml @@ -23,8 +23,7 @@ comment: coverage: # Disable GitHub inline annotations (e.g., red/yellow markers in 'Files changed'). - coverage: - annotations: false + annotations: false status: project: # Post an overall project-level GitHub check summarizing coverage status. From d5f3b3f79a0facf733454269018ef636620a54bd Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 21:32:35 +0000 Subject: [PATCH 44/56] test --- .github/workflows/coverage_tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 5d43c6e80..23c0ae124 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -87,6 +87,9 @@ jobs: # Required to separate coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI. flags: fast name: fast-test-coverage + # Add this to ensure config is used correctly + disable_default_path_fixes: true + - name: Run Slow test and generate report id: run_slow @@ -111,6 +114,9 @@ jobs: files: ./coverage.xml flags: slow name: slow-test-coverage + # Add this to ensure config is used correctly + disable_default_path_fixes: true + - name: Run Superslow test and generate report # Scheduled for Monday (i.e Day 1) or when manually triggered. @@ -134,6 +140,9 @@ jobs: files: ./coverage.xml flags: superslow name: superslow-test-coverage + # Add this to ensure config is used correctly + disable_default_path_fixes: true + # Fail the job in CI if any of the fast/ slow run/ upload steps above failed. - name: Fail if fast/slow test or upload failed From aaf03b45c62b26b579852462a8d32dee5215acfb Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 21:46:46 +0000 Subject: [PATCH 45/56] test --- codecov.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/codecov.yml b/codecov.yml index 4054f3213..cda01574a 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,7 +1,4 @@ flag_management: - default_rules: - # Delay GitHub status checks (e.g., patch/project) until all expected flags are uploaded. - wait_for_results: true individual_flags: - name: fast carryforward: true From feb3d05f42ef7ee25da61b71479c1892ec8fc1e2 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 22:04:16 +0000 Subject: [PATCH 46/56] test --- codecov.yml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/codecov.yml b/codecov.yml index cda01574a..62dea7fc5 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,41 +1,16 @@ -flag_management: - individual_flags: - - name: fast - carryforward: true - - name: slow - carryforward: true - - name: superslow - carryforward: true - -# Enable PR summary comment with coverage details. comment: - # Define the layout of the summary comment to show overall, diff, and file-level stats. layout: "reach, diff, files" - # Overwrite the previous comment instead of posting a new one each time. behavior: default - # Allow comment to appear even if the coverage drop is small or unchanged. require_changes: false - # Disable inline per-line critical path comments in the PR file view. show_critical_paths: false coverage: - # Disable GitHub inline annotations (e.g., red/yellow markers in 'Files changed'). annotations: false status: project: - # Post an overall project-level GitHub check summarizing coverage status. enabled: true - # Use the master branch as the baseline for comparison. target_branch: master - # Compare coverage change relative to the baseline branch. comparator: relative - # Fail the status check if coverage drops by 1% or more. threshold: 1 - # Specify which flags to include in the project-level coverage calculation. - flags: - - fast - - slow - - superslow patch: - # Enable a GitHub check that reports coverage for only the lines changed in the pull request. enabled: true From b8e5bf3cf849af233d89867a9b9b057e06c142d2 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 23:06:16 +0000 Subject: [PATCH 47/56] debug --- codecov.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/codecov.yml b/codecov.yml index 62dea7fc5..b25f2e2dc 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,16 +1,43 @@ +flag_management: + individual_flags: + - name: fast + carryforward: true + - name: slow + carryforward: true + - name: superslow + carryforward: true + +this_field_should_fail: true # ← temporary check + +# Enable PR summary comment with coverage details. comment: + # Define the layout of the summary comment to show overall, diff, and file-level stats. layout: "reach, diff, files" + # Overwrite the previous comment instead of posting a new one each time. behavior: default + # Allow comment to appear even if the coverage drop is small or unchanged. require_changes: false + # Disable inline per-line critical path comments in the PR file view. show_critical_paths: false coverage: + # Disable GitHub inline annotations (e.g., red/yellow markers in 'Files changed'). annotations: false status: project: + # Post an overall project-level GitHub check summarizing coverage status. enabled: true + # Use the master branch as the baseline for comparison. target_branch: master + # Compare coverage change relative to the baseline branch. comparator: relative + # Fail the status check if coverage drops by 1% or more. threshold: 1 + # Specify which flags to include in the project-level coverage calculation. + flags: + - fast + - slow + - superslow patch: + # Enable a GitHub check that reports coverage for only the lines changed in the pull request. enabled: true From e9ce5bfef892beaf2eb0dd48aa4eb1c91853c4d1 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 23:15:10 +0000 Subject: [PATCH 48/56] test --- codecov.yml => .codecov.yml | 2 -- 1 file changed, 2 deletions(-) rename codecov.yml => .codecov.yml (96%) diff --git a/codecov.yml b/.codecov.yml similarity index 96% rename from codecov.yml rename to .codecov.yml index b25f2e2dc..cda01574a 100644 --- a/codecov.yml +++ b/.codecov.yml @@ -7,8 +7,6 @@ flag_management: - name: superslow carryforward: true -this_field_should_fail: true # ← temporary check - # Enable PR summary comment with coverage details. comment: # Define the layout of the summary comment to show overall, diff, and file-level stats. From 760786dde94831b81d6ab46b750ad37150aca0d7 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Wed, 30 Apr 2025 23:25:12 +0000 Subject: [PATCH 49/56] test --- .github/workflows/coverage_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 23c0ae124..308419e56 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,6 +1,9 @@ name: Test coverage on: + push: + branches: + - master pull_request: workflow_dispatch: {} # every day at 00:00 UTC. From 0af1914fdc27ff81f9c576488f2f9fcbf25de5cb Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 1 May 2025 00:07:10 +0000 Subject: [PATCH 50/56] test --- .codecov.yml | 41 ---------------------------- .github/workflows/coverage_tests.yml | 3 -- 2 files changed, 44 deletions(-) delete mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index cda01574a..000000000 --- a/.codecov.yml +++ /dev/null @@ -1,41 +0,0 @@ -flag_management: - individual_flags: - - name: fast - carryforward: true - - name: slow - carryforward: true - - name: superslow - carryforward: true - -# Enable PR summary comment with coverage details. -comment: - # Define the layout of the summary comment to show overall, diff, and file-level stats. - layout: "reach, diff, files" - # Overwrite the previous comment instead of posting a new one each time. - behavior: default - # Allow comment to appear even if the coverage drop is small or unchanged. - require_changes: false - # Disable inline per-line critical path comments in the PR file view. - show_critical_paths: false - -coverage: - # Disable GitHub inline annotations (e.g., red/yellow markers in 'Files changed'). - annotations: false - status: - project: - # Post an overall project-level GitHub check summarizing coverage status. - enabled: true - # Use the master branch as the baseline for comparison. - target_branch: master - # Compare coverage change relative to the baseline branch. - comparator: relative - # Fail the status check if coverage drops by 1% or more. - threshold: 1 - # Specify which flags to include in the project-level coverage calculation. - flags: - - fast - - slow - - superslow - patch: - # Enable a GitHub check that reports coverage for only the lines changed in the pull request. - enabled: true diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 308419e56..23c0ae124 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,9 +1,6 @@ name: Test coverage on: - push: - branches: - - master pull_request: workflow_dispatch: {} # every day at 00:00 UTC. From 023f94efe474801fa9d9f9f450dac3575df13856 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 1 May 2025 00:23:14 +0000 Subject: [PATCH 51/56] checkpoint --- .github/workflows/coverage_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 23c0ae124..c6a086444 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,7 +1,6 @@ name: Test coverage on: - pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: From 1d7701c82789a97df5c99393c60559597dc95e47 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 1 May 2025 01:32:47 +0000 Subject: [PATCH 52/56] checkpoint --- .github/workflows/coverage_tests.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index c6a086444..c77d470d8 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -86,9 +86,6 @@ jobs: # Required to separate coverage reports by type (e.g., fast, slow, superslow) inside the Codecov UI. flags: fast name: fast-test-coverage - # Add this to ensure config is used correctly - disable_default_path_fixes: true - - name: Run Slow test and generate report id: run_slow @@ -113,9 +110,6 @@ jobs: files: ./coverage.xml flags: slow name: slow-test-coverage - # Add this to ensure config is used correctly - disable_default_path_fixes: true - - name: Run Superslow test and generate report # Scheduled for Monday (i.e Day 1) or when manually triggered. @@ -139,9 +133,6 @@ jobs: files: ./coverage.xml flags: superslow name: superslow-test-coverage - # Add this to ensure config is used correctly - disable_default_path_fixes: true - # Fail the job in CI if any of the fast/ slow run/ upload steps above failed. - name: Fail if fast/slow test or upload failed From cf00538e43c7d83f6b6ce301e61e3be6976ef70b Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 1 May 2025 14:43:11 +0000 Subject: [PATCH 53/56] Test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: - 'check_master' passed - 'check_author' passed - 'check_file_size' passed - 'check_python_compile' passed - 'check_gitleaks' passed All checks passed ✅ --- .github/workflows/coverage_tests.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index c77d470d8..459d34904 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -63,19 +63,23 @@ jobs: - name: Run Fast test and generate report id: run_fast continue-on-error: true - env: - GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} - CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} - CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} - CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} - CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} - CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage --suite fast + run: | + echo "Simulating fast test failure" + exit 1 + # env: + # GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} + # CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} + # CSFY_AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + # CSFY_AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} + # CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} + # CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} + # CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} + # run: invoke run_coverage --suite fast - name: Upload Fast Test Coverage to Codecov id: upload_fast - # Upload report if the fast test run successfully and report is generated. + # Only upload if the previous fast test run step succeeded (i.r report generated). + # failed step don’t generate a coverage report, so there's nothing to upload. if: steps.run_fast.outcome == 'success' continue-on-error: true uses: codecov/codecov-action@v5 @@ -102,6 +106,7 @@ jobs: - name: Upload Slow Test Coverage to Codecov id: upload_slow + # Only upload if the previous slow test run step succeeded (i.r report generated). if: steps.run_slow.outcome == 'success' continue-on-error: true uses: codecov/codecov-action@v5 From 8becd022108fc078a37407ff84afa3dbc83a348d Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 1 May 2025 14:57:03 +0000 Subject: [PATCH 54/56] Test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: - 'check_master' passed - 'check_author' passed - 'check_file_size' passed - 'check_python_compile' passed - 'check_gitleaks' passed All checks passed ✅ --- .github/workflows/coverage_tests.yml | 15 +++++++++--- helpers/lib_tasks_pytest.py | 36 ++++++++-------------------- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 459d34904..bc8af9c7e 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,6 +1,7 @@ name: Test coverage on: + pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: @@ -117,8 +118,8 @@ jobs: name: slow-test-coverage - name: Run Superslow test and generate report - # Scheduled for Monday (i.e Day 1) or when manually triggered. - if: (github.event_name == 'schedule' && env.DAY_OF_WEEK == '1') || github.event_name == 'workflow_dispatch' + # Run only on scheduled jobs or manual trigger + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' env: GH_ACTION_ACCESS_TOKEN: ${{ secrets.GH_ACTION_ACCESS_TOKEN }} CSFY_AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} @@ -127,7 +128,15 @@ jobs: CSFY_AWS_DEFAULT_REGION: ${{ env.AWS_DEFAULT_REGION }} CSFY_ECR_BASE_PATH: ghcr.io/${{ github.repository_owner }} CSFY_AWS_S3_BUCKET: ${{ vars.CSFY_AWS_S3_BUCKET }} - run: invoke run_coverage --suite superslow + run: | + # Only run superslow tests if it's Monday or a manual dispatch. + day_of_week=$(date +%u) + if [ "$day_of_week" = "1" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + invoke run_coverage --suite superslow + else + echo "Skipping superslow tests — not Monday and not manually triggered" + exit 0 + fi - name: Upload Superslow Test Coverage to Codecov #TODO(Shaunak): Consider removing it when we turn this workflow into a reusable one. diff --git a/helpers/lib_tasks_pytest.py b/helpers/lib_tasks_pytest.py index 32da831fb..6c9047be9 100644 --- a/helpers/lib_tasks_pytest.py +++ b/helpers/lib_tasks_pytest.py @@ -925,18 +925,16 @@ def _get_inclusion_settings(target_dir: str) -> Tuple[str, Optional[str]]: return include_in_report, exclude_from_report -def _run_coverage( - ctx, - suite: str, - target_dir: str, - generate_html_report: bool = False, -) -> None: +@task +def run_coverage( + ctx, suite: str, target_dir: str = ".", generate_html_report: bool = False +): """ - Run coverage for a given suite (fast/slow/superslow). + Unified task to run coverage for any test suite. - :param suite: one of "fast", "slow" or "superslow" - :param target_dir: coverage target directory - :param generate_html_report: whether to produce HTML output + :param ctx: invoke context + :param suite: suite to run ("fast", "slow", "superslow") + :param target_dir: directory to measure coverage """ hdbg.dassert_in(suite, ("fast", "slow", "superslow")) # Build the command line. @@ -947,7 +945,8 @@ def _run_coverage( # Enable coverage computation. "--coverage", # Specify which directory to test. - "-p", target_dir, + "-p", + target_dir, ] test_cmd = hlitauti.to_multi_line_cmd(test_cmd_parts) # Run the tests under coverage. @@ -980,21 +979,6 @@ def _run_coverage( hlitauti.run(ctx, docker_cmd_) -@task -def run_coverage( - ctx, suite: str, target_dir: str = ".", generate_html_report: bool = False -): - """ - Unified task to run coverage for any test suite. - - :param ctx: invoke context - :param suite: suite to run ("fast", "slow", "superslow") - :param target_dir: directory to measure coverage - """ - hdbg.dassert_in(suite, ("fast", "slow", "superslow")) - _run_coverage(ctx, suite, target_dir, generate_html_report) - - # ############################################################################# # Traceback. # ############################################################################# From 0d0e75a04446df527fbeca884b3faee1092e9c2c Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 1 May 2025 15:06:37 +0000 Subject: [PATCH 55/56] Test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: - 'check_master' passed - 'check_author' passed - 'check_file_size' passed - 'check_python_compile' passed - 'check_gitleaks' passed All checks passed ✅ --- .github/workflows/coverage_tests.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index bc8af9c7e..4fe4b1806 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -151,10 +151,25 @@ jobs: # Fail the job in CI if any of the fast/ slow run/ upload steps above failed. - name: Fail if fast/slow test or upload failed run: | - if [ "${{ steps.run_fast.outcome }}" != "success" ] || \ - [ "${{ steps.run_slow.outcome }}" != "success" ] || \ - [ "${{ steps.upload_fast.outcome }}" != "success" ] || \ - [ "${{ steps.upload_slow.outcome }}" != "success" ]; then + failed="" + if [ "${{ steps.run_fast.outcome }}" != "success" ]; then + echo "Fast test run failed" + failed="true" + fi + if [ "${{ steps.upload_fast.outcome }}" != "success" ]; then + echo "Fast test coverage upload failed" + failed="true" + fi + if [ "${{ steps.run_slow.outcome }}" != "success" ]; then + echo "Slow test run failed" + failed="true" + fi + if [ "${{ steps.upload_slow.outcome }}" != "success" ]; then + echo "Slow test coverage upload failed" + failed="true" + fi + if [ "$failed" = "true" ]; then echo "At least one fast/slow test or upload step failed." exit 1 fi + From ca1dacb70ec678b6ea85326df955e51a4b107552 Mon Sep 17 00:00:00 2001 From: shaunak01 Date: Thu, 1 May 2025 15:42:51 +0000 Subject: [PATCH 56/56] Checkpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: - 'check_master' passed - 'check_author' passed - 'check_file_size' passed - 'check_python_compile' passed - 'check_gitleaks' passed All checks passed ✅ --- .github/workflows/coverage_tests.yml | 2 - .../Master_buildmeister_dashboard.ipynb | 727 +++++++++--------- .../Master_buildmeister_dashboard.py | 18 + 3 files changed, 393 insertions(+), 354 deletions(-) diff --git a/.github/workflows/coverage_tests.yml b/.github/workflows/coverage_tests.yml index 4fe4b1806..1f6e9f2ae 100644 --- a/.github/workflows/coverage_tests.yml +++ b/.github/workflows/coverage_tests.yml @@ -1,7 +1,6 @@ name: Test coverage on: - pull_request: workflow_dispatch: {} # every day at 00:00 UTC. schedule: @@ -172,4 +171,3 @@ jobs: echo "At least one fast/slow test or upload step failed." exit 1 fi - diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb index a0d0169ce..48f2b4c28 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb +++ b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.ipynb @@ -1,354 +1,377 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "e3103ff3", - "metadata": {}, - "source": [ - " TODO(Grisha): does it belong to the `devops` dir?" - ] - }, - { - "cell_type": "markdown", - "id": "9d992fed", - "metadata": {}, - "source": [ - "# Description" - ] - }, - { - "cell_type": "markdown", - "id": "3e381a7d", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-02T08:02:05.889049Z", - "start_time": "2024-02-02T08:02:05.883420Z" - } - }, - "source": [ - "The notebook reports the latest build status for multiple repos." - ] - }, - { - "cell_type": "markdown", - "id": "982f47f1", - "metadata": {}, - "source": [ - "# Imports" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "97bbec36", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.038091Z", - "start_time": "2024-02-07T17:59:42.002068Z" - } - }, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "518df056", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.078514Z", - "start_time": "2024-02-07T17:59:42.041301Z" - } - }, - "outputs": [], - "source": [ - "import logging\n", - "from typing import Dict\n", - "\n", - "import pandas as pd\n", - "from IPython.display import Markdown, display\n", - "\n", - "import helpers.hdbg as hdbg\n", - "import helpers.henv as henv\n", - "import helpers.hpandas as hpandas\n", - "import helpers.hprint as hprint\n", - "import helpers.lib_tasks_gh as hlitagh\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f0793aa5", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.268049Z", - "start_time": "2024-02-07T17:59:42.081426Z" - } - }, - "outputs": [], - "source": [ - "hdbg.init_logger(verbosity=logging.INFO)\n", - "_LOG = logging.getLogger(__name__)\n", - "_LOG.info(\"%s\", henv.get_system_signature()[0])\n", - "hprint.config_notebook()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "93c2d39f", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.338614Z", - "start_time": "2024-02-07T17:59:42.271472Z" - } - }, - "outputs": [], - "source": [ - "# Set the display options to print the full table.\n", - "pd.set_option(\"display.max_colwidth\", None)\n", - "pd.set_option(\"display.max_columns\", None)" - ] - }, - { - "cell_type": "markdown", - "id": "14f379d5", - "metadata": { - "lines_to_next_cell": 2 - }, - "source": [ - "# Utils" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1f41a8dd", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:42.380319Z", - "start_time": "2024-02-07T17:59:42.343492Z" - } - }, - "outputs": [], - "source": [ - "def make_clickable(url: str) -> str:\n", - " \"\"\"\n", - " Wrapper to make the URL value clickable.\n", - "\n", - " :param url: URL value to convert\n", - " :return: clickable URL link\n", - " \"\"\"\n", - " return f'{url}'\n", - "\n", - "\n", - "def color_format(val: str, status_color_mapping: Dict[str, str]) -> str:\n", - " \"\"\"\n", - " Return the color depends on status.\n", - "\n", - " :param val: value of the status e.g. `failure`\n", - " :param status_color_mapping: mapping statuses to the colors e.g.:\n", - " ```\n", - " {\n", - " \"success\": \"green\",\n", - " \"failure\": \"red\",\n", - " }\n", - " ```\n", - " \"\"\"\n", - " if val in status_color_mapping:\n", - " color = status_color_mapping[val]\n", - " else:\n", - " color = \"grey\"\n", - " return f\"background-color: {color}\"" - ] - }, - { - "cell_type": "markdown", - "id": "189f2c75", - "metadata": {}, - "source": [ - "# GH workflows state" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "865bc9f2", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:57.513155Z", - "start_time": "2024-02-07T17:59:42.383039Z" - } - }, - "outputs": [], - "source": [ - "repo_list = [\n", - " \"cryptokaizen/cmamp\",\n", - " \"cryptokaizen/orange\",\n", - " \"cryptokaizen/lemonade\",\n", - " \"causify-ai/kaizenflow\",\n", - "]\n", - "workflow_df = hlitagh.gh_get_details_for_all_workflows(repo_list)\n", - "# Reorder columns.\n", - "columns_order = [\"repo_name\", \"workflow_name\", \"conclusion\", \"url\"]\n", - "workflow_df = workflow_df[columns_order]\n", - "# Make URL values clickable.\n", - "workflow_df[\"url\"] = workflow_df[\"url\"].apply(make_clickable)\n", - "_LOG.info(hpandas.df_to_str(workflow_df, log_level=logging.INFO))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f7e999ce", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:57.585606Z", - "start_time": "2024-02-07T17:59:57.515915Z" - } - }, - "outputs": [], - "source": [ - "status_color_mapping = {\n", - " \"success\": \"green\",\n", - " \"failure\": \"red\",\n", - "}\n", - "repos = workflow_df[\"repo_name\"].unique()\n", - "display(Markdown(\"## Overall Status\"))\n", - "current_timestamp = pd.Timestamp.now(tz=\"America/New_York\")\n", - "display(Markdown(f\"**Last run: {current_timestamp}**\"))\n", - "for repo in repos:\n", - " # Calculate the overall status.\n", - " repo_df = workflow_df[workflow_df[\"repo_name\"] == repo]\n", - " overall_status = hlitagh.gh_get_overall_build_status_for_repo(repo_df)\n", - " display(Markdown(f\"## {repo}: {overall_status}\"))\n", - " repo_df = repo_df.drop(columns=[\"repo_name\"])\n", - " display(\n", - " repo_df.style.map(\n", - " color_format,\n", - " status_color_mapping=status_color_mapping,\n", - " subset=[\"conclusion\"],\n", - " )\n", - " )" - ] - }, - { - "cell_type": "markdown", - "id": "a00870a9", - "metadata": {}, - "source": [ - "# Allure reports" - ] - }, - { - "cell_type": "markdown", - "id": "36e93fca", - "metadata": {}, - "source": [ - "- fast tests: http://172.30.2.44/allure_reports/cmamp/fast/latest/index.html\n", - "- slow tests: http://172.30.2.44/allure_reports/cmamp/slow/latest/index.html\n", - "- superslow tests: http://172.30.2.44/allure_reports/cmamp/superslow/latest/index.html" - ] - }, - { - "cell_type": "markdown", - "id": "bb8ed505", - "metadata": {}, - "source": [ - "# Number of open pull requests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "69dbda1d", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-07T17:59:59.309022Z", - "start_time": "2024-02-07T17:59:57.588291Z" - } - }, - "outputs": [], - "source": [ - "for repo in repo_list:\n", - " number_prs = len(hlitagh.gh_get_open_prs(repo))\n", - " _LOG.info(\"%s: %s\", repo, number_prs)" - ] - }, - { - "cell_type": "markdown", - "id": "ec63cb5e", - "metadata": {}, - "source": [ - "# Code coverage HTML-page" - ] - }, - { - "cell_type": "markdown", - "id": "569f9404", - "metadata": { - "ExecuteTime": { - "end_time": "2024-01-24T14:40:31.379819Z", - "start_time": "2024-01-24T14:40:31.327151Z" - } - }, - "source": [ - "http://172.30.2.44/html_coverage/runner_master/" - ] - }, - { - "cell_type": "markdown", - "id": "027d1b3d", - "metadata": {}, - "source": [ - "# Code Coverage Page - CodeCov" - ] - }, - { - "cell_type": "markdown", - "id": "6c9219e1", - "metadata": {}, - "source": [ - "- Helpers: https://app.codecov.io/gh/causify-ai/helpers" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "CONTENTS:\n", + "- [Description](#description)\n", + "- [Imports](#imports)\n", + "- [Utils](#utils)\n", + "- [GH workflows state](#gh-workflows-state)\n", + "- [Allure reports](#allure-reports)\n", + "- [Number of open pull requests](#number-of-open-pull-requests)\n", + "- [Code coverage HTML-page](#code-coverage-html-page)\n", + "- [Code Coverage Page - CodeCov](#code-coverage-page---codecov)" + ] + }, + { + "cell_type": "markdown", + "id": "e3103ff3", + "metadata": {}, + "source": [ + " TODO(Grisha): does it belong to the `devops` dir?" + ] + }, + { + "cell_type": "markdown", + "id": "9d992fed", + "metadata": {}, + "source": [ + "\n", + "# Description" + ] + }, + { + "cell_type": "markdown", + "id": "3e381a7d", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-02T08:02:05.889049Z", + "start_time": "2024-02-02T08:02:05.883420Z" + } + }, + "source": [ + "The notebook reports the latest build status for multiple repos." + ] + }, + { + "cell_type": "markdown", + "id": "982f47f1", + "metadata": {}, + "source": [ + "\n", + "# Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97bbec36", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.038091Z", + "start_time": "2024-02-07T17:59:42.002068Z" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "518df056", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.078514Z", + "start_time": "2024-02-07T17:59:42.041301Z" + } + }, + "outputs": [], + "source": [ + "import logging\n", + "from typing import Dict\n", + "\n", + "import pandas as pd\n", + "from IPython.display import Markdown, display\n", + "\n", + "import helpers.hdbg as hdbg\n", + "import helpers.henv as henv\n", + "import helpers.hpandas as hpandas\n", + "import helpers.hprint as hprint\n", + "import helpers.lib_tasks_gh as hlitagh\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0793aa5", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.268049Z", + "start_time": "2024-02-07T17:59:42.081426Z" + } + }, + "outputs": [], + "source": [ + "hdbg.init_logger(verbosity=logging.INFO)\n", + "_LOG = logging.getLogger(__name__)\n", + "_LOG.info(\"%s\", henv.get_system_signature()[0])\n", + "hprint.config_notebook()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93c2d39f", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.338614Z", + "start_time": "2024-02-07T17:59:42.271472Z" + } + }, + "outputs": [], + "source": [ + "# Set the display options to print the full table.\n", + "pd.set_option(\"display.max_colwidth\", None)\n", + "pd.set_option(\"display.max_columns\", None)" + ] + }, + { + "cell_type": "markdown", + "id": "14f379d5", + "metadata": { + "lines_to_next_cell": 2 + }, + "source": [ + "\n", + "# Utils" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f41a8dd", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:42.380319Z", + "start_time": "2024-02-07T17:59:42.343492Z" + } + }, + "outputs": [], + "source": [ + "def make_clickable(url: str) -> str:\n", + " \"\"\"\n", + " Wrapper to make the URL value clickable.\n", + "\n", + " :param url: URL value to convert\n", + " :return: clickable URL link\n", + " \"\"\"\n", + " return f'{url}'\n", + "\n", + "\n", + "def color_format(val: str, status_color_mapping: Dict[str, str]) -> str:\n", + " \"\"\"\n", + " Return the color depends on status.\n", + "\n", + " :param val: value of the status e.g. `failure`\n", + " :param status_color_mapping: mapping statuses to the colors e.g.:\n", + " ```\n", + " {\n", + " \"success\": \"green\",\n", + " \"failure\": \"red\",\n", + " }\n", + " ```\n", + " \"\"\"\n", + " if val in status_color_mapping:\n", + " color = status_color_mapping[val]\n", + " else:\n", + " color = \"grey\"\n", + " return f\"background-color: {color}\"" + ] + }, + { + "cell_type": "markdown", + "id": "189f2c75", + "metadata": {}, + "source": [ + "\n", + "# GH workflows state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "865bc9f2", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:57.513155Z", + "start_time": "2024-02-07T17:59:42.383039Z" + } + }, + "outputs": [], + "source": [ + "repo_list = [\n", + " \"cryptokaizen/cmamp\",\n", + " \"cryptokaizen/orange\",\n", + " \"cryptokaizen/lemonade\",\n", + " \"causify-ai/kaizenflow\",\n", + "]\n", + "workflow_df = hlitagh.gh_get_details_for_all_workflows(repo_list)\n", + "# Reorder columns.\n", + "columns_order = [\"repo_name\", \"workflow_name\", \"conclusion\", \"url\"]\n", + "workflow_df = workflow_df[columns_order]\n", + "# Make URL values clickable.\n", + "workflow_df[\"url\"] = workflow_df[\"url\"].apply(make_clickable)\n", + "_LOG.info(hpandas.df_to_str(workflow_df, log_level=logging.INFO))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7e999ce", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:57.585606Z", + "start_time": "2024-02-07T17:59:57.515915Z" + } + }, + "outputs": [], + "source": [ + "status_color_mapping = {\n", + " \"success\": \"green\",\n", + " \"failure\": \"red\",\n", + "}\n", + "repos = workflow_df[\"repo_name\"].unique()\n", + "display(Markdown(\"## Overall Status\"))\n", + "current_timestamp = pd.Timestamp.now(tz=\"America/New_York\")\n", + "display(Markdown(f\"**Last run: {current_timestamp}**\"))\n", + "for repo in repos:\n", + " # Calculate the overall status.\n", + " repo_df = workflow_df[workflow_df[\"repo_name\"] == repo]\n", + " overall_status = hlitagh.gh_get_overall_build_status_for_repo(repo_df)\n", + " display(Markdown(f\"## {repo}: {overall_status}\"))\n", + " repo_df = repo_df.drop(columns=[\"repo_name\"])\n", + " display(\n", + " repo_df.style.map(\n", + " color_format,\n", + " status_color_mapping=status_color_mapping,\n", + " subset=[\"conclusion\"],\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "a00870a9", + "metadata": {}, + "source": [ + "\n", + "# Allure reports" + ] + }, + { + "cell_type": "markdown", + "id": "36e93fca", + "metadata": {}, + "source": [ + "- fast tests: http://172.30.2.44/allure_reports/cmamp/fast/latest/index.html\n", + "- slow tests: http://172.30.2.44/allure_reports/cmamp/slow/latest/index.html\n", + "- superslow tests: http://172.30.2.44/allure_reports/cmamp/superslow/latest/index.html" + ] + }, + { + "cell_type": "markdown", + "id": "bb8ed505", + "metadata": {}, + "source": [ + "\n", + "# Number of open pull requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "69dbda1d", + "metadata": { + "ExecuteTime": { + "end_time": "2024-02-07T17:59:59.309022Z", + "start_time": "2024-02-07T17:59:57.588291Z" + } + }, + "outputs": [], + "source": [ + "for repo in repo_list:\n", + " number_prs = len(hlitagh.gh_get_open_prs(repo))\n", + " _LOG.info(\"%s: %s\", repo, number_prs)" + ] + }, + { + "cell_type": "markdown", + "id": "ec63cb5e", + "metadata": {}, + "source": [ + "\n", + "# Code coverage HTML-page" + ] + }, + { + "cell_type": "markdown", + "id": "569f9404", + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-24T14:40:31.379819Z", + "start_time": "2024-01-24T14:40:31.327151Z" + } + }, + "source": [ + "http://172.30.2.44/html_coverage/runner_master/" + ] + }, + { + "cell_type": "markdown", + "id": "027d1b3d", + "metadata": {}, + "source": [ + "\n", + "# Code Coverage Page - CodeCov" + ] + }, + { + "cell_type": "markdown", + "id": "6c9219e1", + "metadata": {}, + "source": [ + "- Helpers: https://app.codecov.io/gh/causify-ai/helpers" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py index 1bf5e8341..d0095209c 100644 --- a/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py +++ b/dev_scripts_helpers/update_devops_packages/notebooks/Master_buildmeister_dashboard.py @@ -12,12 +12,24 @@ # name: python3 # --- +# CONTENTS: +# - [Description](#description) +# - [Imports](#imports) +# - [Utils](#utils) +# - [GH workflows state](#gh-workflows-state) +# - [Allure reports](#allure-reports) +# - [Number of open pull requests](#number-of-open-pull-requests) +# - [Code coverage HTML-page](#code-coverage-html-page) +# - [Code Coverage Page - CodeCov](#code-coverage-page---codecov) + # TODO(Grisha): does it belong to the `devops` dir? +# # # Description # The notebook reports the latest build status for multiple repos. +# # # Imports # %load_ext autoreload @@ -48,6 +60,7 @@ pd.set_option("display.max_colwidth", None) pd.set_option("display.max_columns", None) +# # # Utils @@ -84,6 +97,7 @@ def color_format(val: str, status_color_mapping: Dict[str, str]) -> str: # - +# # # GH workflows state repo_list = [ @@ -122,22 +136,26 @@ def color_format(val: str, status_color_mapping: Dict[str, str]) -> str: ) ) +# # # Allure reports # - fast tests: http://172.30.2.44/allure_reports/cmamp/fast/latest/index.html # - slow tests: http://172.30.2.44/allure_reports/cmamp/slow/latest/index.html # - superslow tests: http://172.30.2.44/allure_reports/cmamp/superslow/latest/index.html +# # # Number of open pull requests for repo in repo_list: number_prs = len(hlitagh.gh_get_open_prs(repo)) _LOG.info("%s: %s", repo, number_prs) +# # # Code coverage HTML-page # http://172.30.2.44/html_coverage/runner_master/ +# # # Code Coverage Page - CodeCov # - Helpers: https://app.codecov.io/gh/causify-ai/helpers