diff --git a/.cursor/rules/code-style.mdc b/.cursor/rules/code-style.mdc deleted file mode 100644 index ea867d1ee062..000000000000 --- a/.cursor/rules/code-style.mdc +++ /dev/null @@ -1,42 +0,0 @@ ---- -description: Enforces surgical, minimal edit style matching this CI/CD codebase -alwaysApply: true ---- - -# Code Style — Surgical Edits - -## Context - -This is a ClickHouse branch maintained by Altinity. The primary work is CI/CD: GitHub Actions workflows, Python CI scripts, shell scripts, Dockerfiles, YAML configs, and broken-test management across multiple version branches. - -## Core Rules - -1. **Minimal diffs.** Change only the lines required by the task. Never touch surrounding code, whitespace, imports, or formatting. - -2. **No unsolicited refactoring.** Do not rename, restructure, extract helpers, or "clean up" code that isn't part of the request. If a change seems to require restructuring, ask first. - -3. **Prefer safe, reversible changes over clean-ups.** An early `return` with a `NOTE` comment is better than deleting dead code that might be needed later. - -4. **Match existing patterns exactly.** When adding entries to structured files (YAML, workflow definitions, job configs), copy the format of neighbouring entries verbatim. - -5. **Breadcrumb comments only.** When removing or bypassing code, leave a short `# NOTE (strtgbb): ` comment. Do not add explanatory comments, docstrings, or type annotations that weren't asked for. - -6. **Mechanical replication.** When the same fix applies to multiple files (e.g. bumping a timeout), apply the identical minimal change to each file. Do not "improve" one of them while you're there. - -7. **Ask before going wide.** If a task would touch more than ~3 files or change the structure of anything, confirm the scope before proceeding. - -8. **Use surrounding logging conventions.** Do not assume `print()` — look at how the surrounding code logs and use the same mechanism (e.g. `logging`, a custom logger, `Shell.check`, `print`, etc.). - -9. **Merge-friendly changes.** Structure edits to minimize git merge conflicts. Prefer appending to lists over inserting in the middle. Avoid reformatting lines adjacent to the change. Keep hunks small and isolated. - -10. **Never edit generated files directly.** Files starting with `# generated by praktika` are generated — do not edit them. Instead, edit the corresponding source template under the `ci/` directory. The user will regenerate the output. - -## Anti-patterns - -- Rewriting a function to be "cleaner" -- Mixing logging styles (e.g. adding `logging.info()` where `print()` is used nearby, or vice versa) -- Changing quote styles, whitespace, or import order -- Extracting constants, helpers, or classes that don't exist yet -- Deleting dead code instead of short-circuiting past it -- Editing a `# generated by praktika` file instead of its source template -- Inserting into the middle of a list when appending would work diff --git a/.github/actions/create_workflow_report/create_workflow_report.py b/.github/actions/create_workflow_report/create_workflow_report.py index a28435839c77..8daf0fc24f31 100755 --- a/.github/actions/create_workflow_report/create_workflow_report.py +++ b/.github/actions/create_workflow_report/create_workflow_report.py @@ -323,7 +323,7 @@ def get_checks_errors(client: Client, commit_sha: str, branch_name: str): query = f"""{_checks_latest_test_status_cte(commit_sha, branch_name)} SELECT job_status, job_name, status AS test_status, test_name, results_link FROM latest_test_status - WHERE job_status = 'error' AND test_status NOT IN ('OK', 'SKIPPED') + WHERE job_status = 'error' AND test_status NOT IN ('OK', 'SKIPPED', 'BROKEN') ORDER BY job_name, test_name """ return query_dataframe_with_retry(client, query) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 140271fdc03a..6290b2072a51 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -5272,7 +5272,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_ubsan, ast_fuzzer_arm_asan, build_amd_asan, build_amd_binary, build_amd_coverage, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_amd_ubsan, buzzhouse_arm_asan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sqllogic_test, sqltest, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_coverage_1_8, stateless_tests_amd_coverage_2_8, stateless_tests_amd_coverage_3_8, stateless_tests_amd_coverage_4_8, stateless_tests_amd_coverage_5_8, stateless_tests_amd_coverage_6_8, stateless_tests_amd_coverage_7_8, stateless_tests_amd_coverage_8_8, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential_1_2, stateless_tests_arm_asan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan, unit_tests_msan, unit_tests_tsan, unit_tests_ubsan] - if: ${{ always() }} + if: ${{ always() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: data: ${{ steps.run.outputs.DATA }} @@ -5347,7 +5347,7 @@ jobs: secrets: inherit with: runner_type: altinity-regression-tester - commit: c7897a6a858a9ef9c7b3c519e7291cfd3c2ec646 + commit: release arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 210 @@ -5359,7 +5359,7 @@ jobs: secrets: inherit with: runner_type: altinity-regression-tester-aarch64 - commit: c7897a6a858a9ef9c7b3c519e7291cfd3c2ec646 + commit: release arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 210 @@ -5385,7 +5385,7 @@ jobs: data: ${{ needs.config_workflow.outputs.data }} FinishCIReport: - if: ${{ !cancelled() }} + if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} needs: - config_workflow - dockers_build_amd @@ -5525,7 +5525,7 @@ jobs: SourceUpload: needs: [config_workflow, build_amd_release] - if: ${{ !failure() && !cancelled() }} + if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' && !contains(needs.*.outputs.pipeline_status, 'failure') }} runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] env: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9fad377ea76b..2e67e701a4d8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5050,7 +5050,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_ubsan, ast_fuzzer_arm_asan, build_amd_asan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_amd_ubsan, buzzhouse_arm_asan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_asan_targeted, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, quick_functional_tests, sqllogic_test, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential_1_2, stateless_tests_arm_asan_azure_sequential_2_2, stateless_tests_arm_asan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3, unit_tests_asan, unit_tests_msan, unit_tests_tsan, unit_tests_ubsan] - if: ${{ always() }} + if: ${{ always() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: data: ${{ steps.run.outputs.DATA }} @@ -5125,7 +5125,7 @@ jobs: secrets: inherit with: runner_type: altinity-regression-tester - commit: c7897a6a858a9ef9c7b3c519e7291cfd3c2ec646 + commit: release arch: release build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 210 @@ -5137,14 +5137,14 @@ jobs: secrets: inherit with: runner_type: altinity-regression-tester-aarch64 - commit: c7897a6a858a9ef9c7b3c519e7291cfd3c2ec646 + commit: release arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 210 workflow_config: ${{ needs.config_workflow.outputs.data }} FinishCIReport: - if: ${{ !cancelled() }} + if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} needs: - config_workflow - dockers_build_amd @@ -5277,7 +5277,7 @@ jobs: SourceUpload: needs: [config_workflow, build_amd_release] - if: ${{ !failure() && !cancelled() }} + if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' && !contains(needs.*.outputs.pipeline_status, 'failure') }} runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] env: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 4b0e7af9ab57..d30ee15f6520 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -277,7 +277,7 @@ jobs: build_sha: ${{ inputs.build_sha }} set_commit_status: true job_name: iceberg - extra_args: --only ${{ matrix.PART == 1 && '"/iceberg/iceberg engine/rest catalog/*" "/iceberg/s3 table function/*" "/iceberg/icebergS3 table function/*" "/iceberg/iceberg cache/*"' || '"/iceberg/iceberg engine/glue catalog/*" "/iceberg/iceberg table engine/*"' }} + extra_args: --only ${{ matrix.PART == 1 && '"/iceberg/iceberg engine/rest catalog/*" "/iceberg/s3 table function/*" "/iceberg/icebergS3 table function/*" "/iceberg/iceberg cache/*"' || '"/iceberg/iceberg engine/glue catalog/*" "/iceberg/iceberg table engine/*" "/iceberg/export partition/*"' }} secrets: inherit LDAP: if: | diff --git a/.github/workflows/release_builds.yml b/.github/workflows/release_builds.yml index e67180011433..41d2874cbc3f 100644 --- a/.github/workflows/release_builds.yml +++ b/.github/workflows/release_builds.yml @@ -983,7 +983,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] needs: [build_amd_asan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ always() }} + if: ${{ always() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: data: ${{ steps.run.outputs.DATA }} @@ -1073,7 +1073,7 @@ jobs: data: ${{ needs.config_workflow.outputs.data }} FinishCIReport: - if: ${{ !cancelled() }} + if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} needs: - config_workflow - dockers_build_amd @@ -1115,7 +1115,7 @@ jobs: SourceUpload: needs: [config_workflow, build_amd_release] - if: ${{ !failure() && !cancelled() }} + if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' && !contains(needs.*.outputs.pipeline_status, 'failure') }} runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] env: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/ci/jobs/fast_test.py b/ci/jobs/fast_test.py index 0e996147ccb2..90dae093da32 100644 --- a/ci/jobs/fast_test.py +++ b/ci/jobs/fast_test.py @@ -175,6 +175,9 @@ def main(): if info.is_local_run or info.is_community_pr: print("NOTE: Community contribution or local run - set sccache to run without AWS credentials") os.environ["SCCACHE_S3_NO_CREDENTIALS"] = "true" + # NOTE (strtgbb): sccache will throw an error if AWS credentials are present with SCCACHE_S3_NO_CREDENTIALS=1 + os.environ.pop("AWS_SECRET_ACCESS_KEY", None) + os.environ.pop("AWS_ACCESS_KEY_ID", None) else: pass # NOTE (strtgbb): Not used yet, but we should look into setting up the secrets for it diff --git a/ci/jobs/functional_tests.py b/ci/jobs/functional_tests.py index ed24514ad403..84775a71137e 100644 --- a/ci/jobs/functional_tests.py +++ b/ci/jobs/functional_tests.py @@ -363,7 +363,11 @@ def main(): if not has_stateful: has_stateful_tests = False - targeter = Targeting(info=info) + if not info.is_community_pr: + targeter = Targeting(info=info) + else: + targeter = None + if is_flaky_check or is_bugfix_validation: if info.is_local_run: assert ( diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index 6cef596c3110..b4f414eee704 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -35,7 +35,7 @@ class AltinityWorkflowTemplates: echo "Workflow Run Report: [View Report]($REPORT_LINK)" >> $GITHUB_STEP_SUMMARY """ # Additional jobs - REGRESSION_HASH = "c7897a6a858a9ef9c7b3c519e7291cfd3c2ec646" + REGRESSION_HASH = "release" ALTINITY_JOBS = { "GrypeScan": r""" GrypeScanServer: @@ -108,7 +108,7 @@ class AltinityWorkflowTemplates: """, "CIReport": r""" FinishCIReport: - if: ${{ !cancelled() }} + if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} needs: {ALL_JOBS} runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] @@ -127,7 +127,7 @@ class AltinityWorkflowTemplates: "SourceUpload": r""" SourceUpload: needs: [config_workflow, build_amd_release] - if: ${{ !failure() && !cancelled() }} + if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' && !contains(needs.*.outputs.pipeline_status, 'failure') }} runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] env: COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/ci/praktika/yaml_generator.py b/ci/praktika/yaml_generator.py index 15b95f87a7a2..af184588e09d 100644 --- a/ci/praktika/yaml_generator.py +++ b/ci/praktika/yaml_generator.py @@ -243,7 +243,7 @@ class Templates: """ TEMPLATE_IF_EXPRESSION_ALWAYS = """ - if: ${{ always() }}\ + if: ${{ always() && needs.config_workflow.outputs.pipeline_status != '' }}\ """ def __init__(self): diff --git a/programs/keeper/dashboard.html b/programs/keeper/dashboard.html index c927f467c02c..ade77bf6218b 100644 --- a/programs/keeper/dashboard.html +++ b/programs/keeper/dashboard.html @@ -3,8 +3,7 @@ - + ClickHouse Keeper