From ad69ef29d6729b127bfd8d04f98cb4b2f53c4a28 Mon Sep 17 00:00:00 2001 From: Matt Collins Date: Tue, 21 Jul 2026 00:11:02 +0000 Subject: [PATCH] UID2-7482: pin AKS E2E az to Python 3.13 build to avoid Python 3.14 deadlock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `az` bundled on the GitHub runner (azure-cli 2.88.0) runs on Python 3.14, where msal's lazy `import requests` on the CLI's long-running-operation poller thread deadlocks (_frozen_importlib._DeadlockError) and crashes long-running commands such as `az network vnet create`, `az aks create`, and `az group delete`. This fails the Azure AKS E2E job deterministically. It is an upstream azure-cli defect — Python 3.14 is not yet supported: Azure/azure-cli#32869. azure-cli 2.87.0 is the last release built on Python 3.13, so pin the runner's official (apt) az to it via a new pin_azure_cli composite action, invoked as an explicit setup step in both AKS jobs: e2e-test (before Start AKS cluster) and e2e-test-cleanup (before Stop AKS private operator, which runs `az group delete` on its own runner and would deadlock too). The pinned version is a workflow-level env var (AZURE_CLI_PINNED_VERSION) passed to the action, so it lives with the pipeline config and is bumped in one place. apt makes the pin idempotent (a no-op once pinned), and a failed pin aborts the step (set -e) rather than silently running on the broken az. Remove the action, its two workflow steps, and the env var once azure-cli supports Python 3.14. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/shared-run-e2e-tests.yaml | 16 ++++++++++++ actions/pin_azure_cli/action.yaml | 29 +++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 actions/pin_azure_cli/action.yaml diff --git a/.github/workflows/shared-run-e2e-tests.yaml b/.github/workflows/shared-run-e2e-tests.yaml index 0277ce5a..79aaa621 100644 --- a/.github/workflows/shared-run-e2e-tests.yaml +++ b/.github/workflows/shared-run-e2e-tests.yaml @@ -76,6 +76,10 @@ on: env: REGISTRY: ghcr.io + # Last azure-cli built on Python 3.13; works around the Python 3.14 az deadlock + # (Azure/azure-cli#32869). Bump or remove once azure-cli supports Python 3.14. + AZURE_CLI_PINNED_VERSION: "2.87.0" + E2E_UID2_INTEG_GCP_ARGS_JSON: ${{ secrets.E2E_UID2_INTEG_GCP_ARGS_JSON }} E2E_UID2_INTEG_AWS_ARGS_JSON: ${{ secrets.E2E_UID2_INTEG_AWS_ARGS_JSON }} E2E_UID2_INTEG_AZURE_ARGS_JSON: ${{ secrets.E2E_UID2_INTEG_AZURE_ARGS_JSON }} @@ -204,6 +208,12 @@ jobs: target_environment: ${{ inputs.target_environment }} aws_pcr0: ${{ inputs.aws_pcr0 }} + - name: Pin Azure CLI (Azure/azure-cli#32869) + if: ${{ inputs.operator_type == 'aks' }} + uses: IABTechLab/uid2-shared-actions/actions/pin_azure_cli@v3 + with: + azure_cli_version: ${{ env.AZURE_CLI_PINNED_VERSION }} + - name: Start AKS cluster id: start_aks_cluster if: ${{ inputs.operator_type == 'aks' }} @@ -404,6 +414,12 @@ jobs: aws_stack_name: ${{ needs.e2e-test.outputs.aws_stack_name }} aws_region: ${{ inputs.aws_region }} + - name: Pin Azure CLI (Azure/azure-cli#32869) + if: ${{ inputs.operator_type == 'aks' }} + uses: IABTechLab/uid2-shared-actions/actions/pin_azure_cli@v3 + with: + azure_cli_version: ${{ env.AZURE_CLI_PINNED_VERSION }} + - name: Stop AKS private operator if: ${{ inputs.operator_type == 'aks' }} uses: IABTechLab/uid2-shared-actions/actions/stop_aks_private_operator@v3 diff --git a/actions/pin_azure_cli/action.yaml b/actions/pin_azure_cli/action.yaml new file mode 100644 index 00000000..f66f1f77 --- /dev/null +++ b/actions/pin_azure_cli/action.yaml @@ -0,0 +1,29 @@ +name: Pin Azure CLI +description: > + Pin the runner's Azure CLI to a specific version via apt (downgrading if + needed), so az runs on the Python interpreter that version bundles. Works + around Azure/azure-cli#32869: on Python 3.14, msal's lazy `import requests` on + the CLI's long-running-operation poller thread deadlocks + (_frozen_importlib._DeadlockError) and crashes commands such as + `az network vnet create`, `az aks create`, and `az group delete`. Pin to a + build on Python <= 3.13 until upstream ships Python 3.14 support. + +inputs: + azure_cli_version: + description: azure-cli version to pin; that version's apt build is installed, e.g. "2.87.0". + required: true + +runs: + using: "composite" + steps: + - name: Pin azure-cli + shell: bash + env: + AZURE_CLI_VERSION: ${{ inputs.azure_cli_version }} + run: | + : "${AZURE_CLI_VERSION:?azure_cli_version input is required}" + # Idempotent: a no-op when az is already at the pinned version. The glob + # matches the -1~ apt revision so it need not be spelled out. + pkg="azure-cli=${AZURE_CLI_VERSION}*" + sudo apt-get update + sudo apt-get install -y --allow-downgrades "${pkg}"