From 89f84015f2550529c17c250fcbfefa857d31e033 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 16:10:24 +0100 Subject: [PATCH 01/16] Add catalog signing of PyManager index files --- windows-release/merge-and-upload.py | 50 +++++++ windows-release/sign-files.yml | 158 +++++++++++++------- windows-release/stage-publish-pymanager.yml | 19 ++- 3 files changed, 175 insertions(+), 52 deletions(-) diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py index 87bbaf6a..71debf62 100644 --- a/windows-release/merge-and-upload.py +++ b/windows-release/merge-and-upload.py @@ -20,6 +20,7 @@ UPLOAD_USER = os.getenv("UPLOAD_USER", "") NO_UPLOAD = os.getenv("NO_UPLOAD", "no")[:1].lower() in "yt1" LOCAL_INDEX = os.getenv("LOCAL_INDEX", "no")[:1].lower() in "yt1" +SIGN_COMMAND = os.getenv("SIGN_COMMAND", "") def find_cmd(env, exe): @@ -40,6 +41,7 @@ def find_cmd(env, exe): PLINK = find_cmd("PLINK", "plink.exe") PSCP = find_cmd("PSCP", "pscp.exe") +MAKECAT = find_cmd("MAKECAT", "makecat.exe") def _std_args(cmd): @@ -193,6 +195,42 @@ def calculate_uploads(): ) +def sign_json(cat_file, *files): + if not MAKECAT: + if not UPLOAD_HOST or NO_UPLOAD: + print("makecat.exe not found, but not uploading, so skip signing.") + return + raise RuntimeError("No makecat.exe found") + if not SIGN_COMMAND: + if not UPLOAD_HOST or NO_UPLOAD: + print("No signing command set, but not uploading, so skip signing.") + return + raise RuntimeError("No SIGN_COMMAND set") + + cat = Path(cat_file).absolute() + cdf = cat.with_suffix(".cdf") + cdf.parent.mkdir(parents=True, exist_ok=True) + + with open(cdf, "w", encoding="ansi") as f: + print("[CatalogHeader]", file=f) + print("Name=", cat.name, sep="", file=f) + print("ResultDir=", cat.parent, sep="", file=f) + print("PublicVersion=0x00000001", file=f) + print("CatalogVersion=2", file=f) + print("HashAlgorithms=SHA256", file=f) + print("EncodingType=", file=f) + print(file=f) + print("[CatalogFiles]", file=f) + for a in map(Path, files): + print("", a.name, "=", a.absolute(), sep="", file=f) + + _run(MAKECAT, "-v", cdf) + if not cat.is_file(): + raise FileNotFoundError(cat) + _run(SIGN_COMMAND, cat) + cdf.unlink() + + def remove_and_insert(index, new_installs): new = {(i["id"].casefold(), i["sort-version"].casefold()) for i in new_installs} to_remove = [ @@ -274,9 +312,11 @@ def find_missing_from_index(url, installs): except FileNotFoundError: pass + print(INDEX_PATH, "mtime =", INDEX_MTIME) + new_installs = [trim_install(i) for i, *_ in UPLOADS] validate_new_installs(new_installs) new_installs = sorted(new_installs, key=install_sortkey) @@ -284,10 +324,16 @@ def find_missing_from_index(url, installs): if INDEX_FILE: INDEX_FILE = Path(INDEX_FILE).absolute() + INDEX_CAT_FILE = INDEX_FILE.with_suffix(".cat") INDEX_FILE.parent.mkdir(parents=True, exist_ok=True) with open(INDEX_FILE, "w", encoding="utf-8") as f: json.dump(index, f) + sign_json(INDEX_CAT_FILE, INDEX_FILE) +else: + INDEX_CAT_FILE = None + + if MANIFEST_FILE: # Use the sort-version so that the manifest name includes prerelease marks MANIFEST_FILE = Path(MANIFEST_FILE).absolute() @@ -333,6 +379,10 @@ def find_missing_from_index(url, installs): print("Uploading", INDEX_FILE, "to", INDEX_URL) upload_ssh(INDEX_FILE, INDEX_PATH) + if INDEX_CAT_FILE: + print("Uploading", INDEX_CAT_FILE, "to", f"{INDEX_URL}.cat") + upload_ssh(INDEX_CAT_FILE, f"{INDEX_PATH}.cat") + print("Purging", len(UPLOADS), "uploaded files") parents = set() for i, *_ in UPLOADS: diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index 6809d9c9..abbcd523 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -6,33 +6,83 @@ parameters: ExtractDir: '' SigningCertificate: '' ExportCommand: '' + ExportLegacyCommand: '' ContinueOnError: false + InstallTool: true + InstallLegacyTool: false AzureServiceConnectionName: 'Python Signing' steps: -- ${{ if parameters.SigningCertificate }}: - - powershell: | - # Install sign tool - dotnet tool install --global --prerelease sign - $signtool = (gcm sign -EA SilentlyContinue).Source - if (-not $signtool) { - $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName - } - $signargs = 'code trusted-signing -v Information ' + ` - '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + ` - '-tse "$(TrustedSigningUri)" -tsa "$(TrustedSigningAccount)" -tscp "$(TrustedSigningCertificateName)" ' + ` - '-d "$(SigningDescription)" ' +- ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}: + - ${{ if eq(parameters.InstallTool, 'true') }}: + - powershell: | + # Install sign tool + dotnet tool install --global --prerelease sign + $signtool = (gcm sign -EA SilentlyContinue).Source + if (-not $signtool) { + $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName + } + $signargs = 'code artifact-signing -v Information ' + ` + '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + ` + "-tse ""${env:TSE}"" -tsa ""${env:TSA}"" -tscp ""${env:TSCP}"" -d ""${env:DESCRIPTION}""" + + Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" + Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" + if ($env:EXPORT_COMMAND) { + $signcmd = """$signtool"" $signargs" + Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" + } + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Install Azure Artifact Signing tools' + env: + TSE: $(TrustedSigningUri) + TSA: $(TrustedSigningAccount) + TSCP: $(TrustedSigningCertificateName) + DESCRIPTION: $(SigningDescription) + EXPORT_COMMAND: ${{ parameters.ExportCommand }} + + - ${{ if eq(parameters.InstallLegacyTool, 'true') }}: + - powershell: | + git clone https://github.com/python/cpython-bin-deps --revision fb06137dccc43ed5b030cdd9e3560990b37f39da --depth 1 --progress -v "signtool" + + $signtool = gi signtool\x64\signtool.exe + $dlib = gi signtool\azure_trusted_signing\x64\Azure.CodeSigning.Dlib.dll + Write-Host "##vso[task.setvariable variable=MAKECAT]$(gi signtool\x64\makecat.exe)" + + ConvertTo-Json @{ + Endpoint=$env:TSE; + CodeSigningAccountName=$env:TSA; + CertificateProfileName=$env:TSCP; + # Only allow Azure CLI credentials and environment credentials + ExcludeCredentials=@( + "ManagedIdentityCredential", + "WorkloadIdentityCredential", + "SharedTokenCacheCredential", + "VisualStudioCredential", + "VisualStudioCodeCredential", + "AzurePowerShellCredential", + "AzureDeveloperCliCredential", + "InteractiveBrowserCredential" + ); + } | Out-File signtool\metadata.json -Encoding ascii + Write-Host "##vso[task.setvariable variable=SIGNTOOL_METADATA]$(gi signtool\metadata.json)" - Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" - Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" - if ($env:EXPORT_COMMAND) { - $signcmd = """$signtool"" $signargs" - Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" - } - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Install Trusted Signing tools' - env: - EXPORT_COMMAND: ${{ parameters.ExportCommand }} + $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td SHA256' + ` + "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)""" + Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" + Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" + + if ($env:EXPORT_COMMAND) { + $signcmd = """$signtool"" $signargs" + Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" + } + workingDirectory: $(Pipeline.Workspace) + displayName: 'Download signtool binaries' + env: + TSE: $(TrustedSigningUri) + TSA: $(TrustedSigningAccount) + TSCP: $(TrustedSigningCertificateName) + EXPORT_COMMAND: ${{ parameters.ExportLegacyCommand }} - ${{ if parameters.AzureServiceConnectionName }}: # We sign in once with the AzureCLI task, as it uses OIDC to obtain a @@ -45,25 +95,25 @@ steps: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - "##vso[task.setvariable variable=AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}" - "##vso[task.setvariable variable=AZURE_ID_TOKEN;issecret=true]${env:idToken}" - "##vso[task.setvariable variable=AZURE_TENANT_ID;issecret=true]${env:tenantId}" + "##vso[task.setvariable variable=__AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}" + "##vso[task.setvariable variable=__AZURE_ID_TOKEN;issecret=true]${env:idToken}" + "##vso[task.setvariable variable=__AZURE_TENANT_ID;issecret=true]${env:tenantId}" addSpnToEnvironment: true - powershell: > az login --service-principal - -u $(AZURE_CLIENT_ID) - --tenant $(AZURE_TENANT_ID) + -u $(__AZURE_CLIENT_ID) + --tenant $(__AZURE_TENANT_ID) --allow-no-subscriptions - --federated-token $(AZURE_ID_TOKEN) + --federated-token $(__AZURE_ID_TOKEN) displayName: 'Authenticate signing tools (2/2)' - ${{ if parameters.Include }}: - powershell: | - if ("${{ parameters.Exclude }}") { - $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File) + if ($env:EXCLUDE) { + $files = (dir $env:INCLUDE -Exclude $env:EXCLUDE -File) } else { - $files = (dir ${{ parameters.Include }} -File) + $files = (dir $env:INCLUDE -File) } if ($env:FILTER) { ($env:FILTER -split ';') -join "`n" | Out-File __filelist.txt -Encoding utf8 @@ -82,31 +132,37 @@ steps: continueOnError: true workingDirectory: ${{ parameters.WorkingDir }} env: + INCLUDE: ${{ parameters.Include }} + EXCLUDE: ${{ parameters.Exclude }} TRUSTED_SIGNING_CMD: $(__TrustedSigningCmd) TRUSTED_SIGNING_ARGS: $(__TrustedSigningArgs) ${{ if parameters.Filter }}: FILTER: ${{ parameters.Filter }} -- ${{ if parameters.ExtractDir }}: - - powershell: | - if ("${{ parameters.Exclude }}") { - $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File) - } else { - $files = (dir ${{ parameters.Include }} -File) - } - $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1 - if (-not $c) { - Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}" - exit - } + - ${{ if parameters.ExtractDir }}: + - powershell: | + if ($env:EXCLUDE) { + $files = (dir $env:INCLUDE -Exclude $env:EXCLUDE -File) + } else { + $files = (dir $env:INCLUDE -File) + } + $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}" + exit + } - $d = mkdir "${{ parameters.ExtractDir }}" -Force - $cf = "$d\cert.cer" - [IO.File]::WriteAllBytes($cf, $c.RawData) - $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower() + $d = mkdir $env:EXTRACT_DIR -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.RawData) + $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower() - $info = @{ Subject=$c.Subject; SHA256=$csha; } - $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" - displayName: "Extract certificate info" - workingDirectory: ${{ parameters.WorkingDir }} + $info = @{ Subject=$c.Subject; SHA256=$csha; } + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + workingDirectory: ${{ parameters.WorkingDir }} + env: + INCLUDE: ${{ parameters.Include }} + EXCLUDE: ${{ parameters.Exclude }} + EXTRACT_DIR: ${{ parameters.ExtractDir }} diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml index f24f5073..034828b7 100644 --- a/windows-release/stage-publish-pymanager.yml +++ b/windows-release/stage-publish-pymanager.yml @@ -3,6 +3,7 @@ parameters: DoFreethreaded: false DoEmbed: false HashAlgorithms: ['SHA256'] + SigningCertificate: '' Artifacts: - name: win32 @@ -31,6 +32,10 @@ jobs: variables: - group: PythonOrgPublish + - ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: + - group: CPythonSign + - ${{ if eq(parameters.SigningCertificate, 'TestSign') }}: + - group: CPythonTestSign workspace: clean: all @@ -64,11 +69,20 @@ jobs: displayName: 'Download PuTTY key' - powershell: | - git clone https://github.com/python/cpython-bin-deps --branch putty --single-branch --depth 1 --progress -v "putty" + git clone https://github.com/python/cpython-bin-deps --revision 9f9e6fc31a55406ee5ff0198ea47bbb445eeb942 --depth 1 --progress -v "putty" "##vso[task.prependpath]$(gi putty)" workingDirectory: $(Pipeline.Workspace) displayName: 'Download PuTTY binaries' + # Use the template just to configure the signing tool. + - template: sign-files.yml + parameters: + Include: "" + InstallTool: false + InstallLegacyTool: true + ExportLegacyCommand: SIGN_COMMAND + SigningCertificate: ${{ parameters.SigningCertificate }} + - powershell: | if ($env:FILENAME) { "##vso[task.setvariable variable=_PyManagerIndexFilename]${env:FILENAME}" @@ -113,6 +127,9 @@ jobs: UPLOAD_HOST_KEY: $(PyDotOrgHostKey) UPLOAD_USER: $(PyDotOrgUsername) UPLOAD_KEYFILE: $(sshkey.secureFilePath) + ${{ if variables['SIGN_COMMAND'] }}: + MAKECAT: $(MAKECAT) + SIGN_COMMAND: $(SIGN_COMMAND) - ${{ each alg in parameters.HashAlgorithms }}: - powershell: | From 6e3043c19d8bb31817e0b0431254da5999ea5d3e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 16:14:15 +0100 Subject: [PATCH 02/16] Run signing for test publish --- windows-release/stage-publish-pymanager.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml index 034828b7..02b63463 100644 --- a/windows-release/stage-publish-pymanager.yml +++ b/windows-release/stage-publish-pymanager.yml @@ -108,6 +108,9 @@ jobs: INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' + ${{ if variables['SIGN_COMMAND'] }}: + MAKECAT: $(MAKECAT) + SIGN_COMMAND: $(SIGN_COMMAND) - powershell: | "Uploading following packages:" From f5b9872a489ea84dbe5946010aa2ba11312626e3 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 16:28:08 +0100 Subject: [PATCH 03/16] Make testing easier --- windows-release/merge-and-upload.py | 5 +- windows-release/stage-publish-pymanager.yml | 85 +++++++++++---------- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py index 71debf62..06965693 100644 --- a/windows-release/merge-and-upload.py +++ b/windows-release/merge-and-upload.py @@ -26,7 +26,10 @@ def find_cmd(env, exe): cmd = os.getenv(env) if cmd: - return Path(cmd) + cmd = Path(cmd) + if not cmd.is_file(): + raise RuntimeError(f"Could not find {cmd} to perform upload. Incorrect %{env}% setting.") + return cmd for p in os.getenv("PATH", "").split(";"): if p: cmd = Path(p) / exe diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml index 02b63463..e08de360 100644 --- a/windows-release/stage-publish-pymanager.yml +++ b/windows-release/stage-publish-pymanager.yml @@ -62,11 +62,12 @@ jobs: artifact: pymanager_${{ a.name }} targetPath: $(Build.BinariesDirectory)\${{ a.name }} - - task: DownloadSecureFile@1 - name: sshkey - inputs: - secureFile: pydotorg-ssh.ppk - displayName: 'Download PuTTY key' + - ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: + - task: DownloadSecureFile@1 + name: sshkey + inputs: + secureFile: pydotorg-ssh.ppk + displayName: 'Download PuTTY key' - powershell: | git clone https://github.com/python/cpython-bin-deps --revision 9f9e6fc31a55406ee5ff0198ea47bbb445eeb942 --depth 1 --progress -v "putty" @@ -95,44 +96,44 @@ jobs: FILENAME: $(PyManagerIndexFilename) displayName: 'Infer index filename' - - powershell: | - "Uploading following packages:" - (dir "__install__.*.json").FullName - (dir "*\__install__.json").FullName - python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" - workingDirectory: $(Build.BinariesDirectory) - condition: and(succeeded(), ne(variables['IsRealSigned'], 'true')) - displayName: 'Produce uploadable ZIPs' - env: - NO_UPLOAD: 1 - INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' - INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' - MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' - ${{ if variables['SIGN_COMMAND'] }}: - MAKECAT: $(MAKECAT) - SIGN_COMMAND: $(SIGN_COMMAND) + - ${{ if ne(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: + - powershell: | + "Preparing following packages:" + (dir "__install__.*.json").FullName + (dir "*\__install__.json").FullName + python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Produce uploadable ZIPs (no upload)' + env: + NO_UPLOAD: 1 + INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' + INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' + MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' + ${{ if variables['SIGN_COMMAND'] }}: + MAKECAT: $(MAKECAT) + SIGN_COMMAND: $(SIGN_COMMAND) - - powershell: | - "Uploading following packages:" - (dir "__install__.*.json").FullName - (dir "*\__install__.json").FullName - python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" - workingDirectory: $(Build.BinariesDirectory) - condition: and(succeeded(), eq(variables['IsRealSigned'], 'true')) - displayName: 'Upload ZIPs' - env: - INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' - INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' - MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' - UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix) - UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix) - UPLOAD_HOST: $(PyDotOrgServer) - UPLOAD_HOST_KEY: $(PyDotOrgHostKey) - UPLOAD_USER: $(PyDotOrgUsername) - UPLOAD_KEYFILE: $(sshkey.secureFilePath) - ${{ if variables['SIGN_COMMAND'] }}: - MAKECAT: $(MAKECAT) - SIGN_COMMAND: $(SIGN_COMMAND) + - ${{ else }}: + - powershell: | + "Uploading following packages:" + (dir "__install__.*.json").FullName + (dir "*\__install__.json").FullName + python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Upload ZIPs' + env: + INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' + INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' + MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' + UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix) + UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix) + UPLOAD_HOST: $(PyDotOrgServer) + UPLOAD_HOST_KEY: $(PyDotOrgHostKey) + UPLOAD_USER: $(PyDotOrgUsername) + UPLOAD_KEYFILE: $(sshkey.secureFilePath) + ${{ if variables['SIGN_COMMAND'] }}: + MAKECAT: $(MAKECAT) + SIGN_COMMAND: $(SIGN_COMMAND) - ${{ each alg in parameters.HashAlgorithms }}: - powershell: | From b5df5fec3eb32d17ca85fc6d489544e58e2e8b5f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 16:32:31 +0100 Subject: [PATCH 04/16] Pass through SigningCertificate --- windows-release/azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/windows-release/azure-pipelines.yml b/windows-release/azure-pipelines.yml index f4e08765..55d99987 100644 --- a/windows-release/azure-pipelines.yml +++ b/windows-release/azure-pipelines.yml @@ -320,6 +320,7 @@ stages: BuildToPublish: ${{ parameters.BuildToPublish }} DoEmbed: ${{ parameters.DoEmbed }} DoFreethreaded: ${{ parameters.DoFreethreaded }} + SigningCertificate: ${{ iif(eq(parameters.SigningCertificate, 'Unsigned'), '', parameters.SigningCertificate) }} - ${{ if eq(parameters.DoMSI, 'true') }}: - template: stage-publish-pythonorg.yml parameters: From 0b469ebaf3fa157cf990e9e14f774d40db05ef4d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 16:37:11 +0100 Subject: [PATCH 05/16] Pass Popen argument correctly --- windows-release/merge-and-upload.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py index 06965693..2710a505 100644 --- a/windows-release/merge-and-upload.py +++ b/windows-release/merge-and-upload.py @@ -65,7 +65,9 @@ class RunError(Exception): pass -def _run(*args): +def _run(*args, single_cmd=False): + if single_cmd: + args = args[0] with subprocess.Popen( args, stdout=subprocess.PIPE, @@ -230,7 +232,8 @@ def sign_json(cat_file, *files): _run(MAKECAT, "-v", cdf) if not cat.is_file(): raise FileNotFoundError(cat) - _run(SIGN_COMMAND, cat) + # Pass as a single arg because the command variable has its own arguments + _run(f'{SIGN_COMMAND} "{cat}"', single_cmd=True) cdf.unlink() From b555de09d81ff894cae792377da6702ce05a4797 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 16:47:06 +0100 Subject: [PATCH 06/16] Case sensitive? --- windows-release/sign-files.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index abbcd523..8133bda5 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -67,7 +67,7 @@ steps: } | Out-File signtool\metadata.json -Encoding ascii Write-Host "##vso[task.setvariable variable=SIGNTOOL_METADATA]$(gi signtool\metadata.json)" - $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td SHA256' + ` + $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td sha256' + ` "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)""" Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" From 983c5409a2718db879fdc52cf3b30a6e9b977a1b Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 16:51:45 +0100 Subject: [PATCH 07/16] Nope, missing a space --- windows-release/sign-files.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index 8133bda5..314ca4e7 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -67,7 +67,7 @@ steps: } | Out-File signtool\metadata.json -Encoding ascii Write-Host "##vso[task.setvariable variable=SIGNTOOL_METADATA]$(gi signtool\metadata.json)" - $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td sha256' + ` + $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td sha256 ' + ` "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)""" Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" From bee864576977a0ec51023617ad04580c6f4bdd69 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 17:20:31 +0100 Subject: [PATCH 08/16] Update sign arguments --- windows-release/sign-files.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index 314ca4e7..e081d6a7 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -24,7 +24,8 @@ steps: } $signargs = 'code artifact-signing -v Information ' + ` '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + ` - "-tse ""${env:TSE}"" -tsa ""${env:TSA}"" -tscp ""${env:TSCP}"" -d ""${env:DESCRIPTION}""" + "-ase ""${env:ASE}"" -asa ""${env:ASA}"" -ascp ""${env:ASCP}"" " + ` + "-act azure-cli -d ""${env:DESCRIPTION}""" Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" @@ -35,9 +36,9 @@ steps: workingDirectory: $(Build.BinariesDirectory) displayName: 'Install Azure Artifact Signing tools' env: - TSE: $(TrustedSigningUri) - TSA: $(TrustedSigningAccount) - TSCP: $(TrustedSigningCertificateName) + ASE: $(TrustedSigningUri) + ASA: $(TrustedSigningAccount) + ASCP: $(TrustedSigningCertificateName) DESCRIPTION: $(SigningDescription) EXPORT_COMMAND: ${{ parameters.ExportCommand }} @@ -53,11 +54,12 @@ steps: Endpoint=$env:TSE; CodeSigningAccountName=$env:TSA; CertificateProfileName=$env:TSCP; - # Only allow Azure CLI credentials and environment credentials + # Only allow Azure CLI credentials ExcludeCredentials=@( "ManagedIdentityCredential", "WorkloadIdentityCredential", "SharedTokenCacheCredential", + "EnvironmentCredential", "VisualStudioCredential", "VisualStudioCodeCredential", "AzurePowerShellCredential", From c6288b9bd05a0d238c984d5a27e4be60f71d6815 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 21:16:13 +0100 Subject: [PATCH 09/16] Satisfy linter --- windows-release/merge-and-upload.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py index 2710a505..3403b7bb 100644 --- a/windows-release/merge-and-upload.py +++ b/windows-release/merge-and-upload.py @@ -28,7 +28,9 @@ def find_cmd(env, exe): if cmd: cmd = Path(cmd) if not cmd.is_file(): - raise RuntimeError(f"Could not find {cmd} to perform upload. Incorrect %{env}% setting.") + raise RuntimeError( + f"Could not find {cmd} to perform upload. Incorrect %{env}% setting." + ) return cmd for p in os.getenv("PATH", "").split(";"): if p: From 328cc1d0815936ef6bb65891b97bb96144c4c49c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 21:18:18 +0100 Subject: [PATCH 10/16] Satisfy linter --- windows-release/merge-and-upload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py index 3403b7bb..62536b03 100644 --- a/windows-release/merge-and-upload.py +++ b/windows-release/merge-and-upload.py @@ -324,7 +324,6 @@ def find_missing_from_index(url, installs): print(INDEX_PATH, "mtime =", INDEX_MTIME) - new_installs = [trim_install(i) for i, *_ in UPLOADS] validate_new_installs(new_installs) new_installs = sorted(new_installs, key=install_sortkey) From b6b191cc9e2b147da990538b4452986a46639e73 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 22:11:38 +0100 Subject: [PATCH 11/16] Fix signing filters --- windows-release/sign-files.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index e081d6a7..2879d2f2 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -113,12 +113,12 @@ steps: - ${{ if parameters.Include }}: - powershell: | if ($env:EXCLUDE) { - $files = (dir $env:INCLUDE -Exclude $env:EXCLUDE -File) + $files = (dir ($env:INCLUDE -split ';').Trim() -Exclude ($env:EXCLUDE -split ';').Trim() -File) } else { - $files = (dir $env:INCLUDE -File) + $files = (dir ($env:INCLUDE -split ';').Trim() -File) } if ($env:FILTER) { - ($env:FILTER -split ';') -join "`n" | Out-File __filelist.txt -Encoding utf8 + ($env:FILTER -split ';').Trim() -join "`n" | Out-File __filelist.txt -Encoding utf8 } else { "*" | Out-File __filelist.txt -Encoding utf8 } @@ -145,9 +145,9 @@ steps: - ${{ if parameters.ExtractDir }}: - powershell: | if ($env:EXCLUDE) { - $files = (dir $env:INCLUDE -Exclude $env:EXCLUDE -File) + $files = (dir ($env:INCLUDE -split ',').Trim() -Exclude ($env:EXCLUDE -split ',').Trim() -File) } else { - $files = (dir $env:INCLUDE -File) + $files = (dir ($env:INCLUDE -split ',').Trim() -File) } $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1 if (-not $c) { From 5caa78e8702ce59b3c0b9e8d1e449d5a3061bc4b Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 22:39:19 +0100 Subject: [PATCH 12/16] Minor YAML fixes --- windows-release/sign-files.yml | 4 ++-- windows-release/stage-publish-pymanager.yml | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index 2879d2f2..bf41b5de 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -113,9 +113,9 @@ steps: - ${{ if parameters.Include }}: - powershell: | if ($env:EXCLUDE) { - $files = (dir ($env:INCLUDE -split ';').Trim() -Exclude ($env:EXCLUDE -split ';').Trim() -File) + $files = (dir ($env:INCLUDE -split ',').Trim() -Exclude ($env:EXCLUDE -split ',').Trim() -File) } else { - $files = (dir ($env:INCLUDE -split ';').Trim() -File) + $files = (dir ($env:INCLUDE -split ',').Trim() -File) } if ($env:FILTER) { ($env:FILTER -split ';').Trim() -join "`n" | Out-File __filelist.txt -Encoding utf8 diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml index e08de360..46a0fde8 100644 --- a/windows-release/stage-publish-pymanager.yml +++ b/windows-release/stage-publish-pymanager.yml @@ -76,6 +76,7 @@ jobs: displayName: 'Download PuTTY binaries' # Use the template just to configure the signing tool. + # This will set MAKECAT and SIGN_COMMAND to be injected into later build steps - template: sign-files.yml parameters: Include: "" @@ -109,9 +110,6 @@ jobs: INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' - ${{ if variables['SIGN_COMMAND'] }}: - MAKECAT: $(MAKECAT) - SIGN_COMMAND: $(SIGN_COMMAND) - ${{ else }}: - powershell: | @@ -131,9 +129,6 @@ jobs: UPLOAD_HOST_KEY: $(PyDotOrgHostKey) UPLOAD_USER: $(PyDotOrgUsername) UPLOAD_KEYFILE: $(sshkey.secureFilePath) - ${{ if variables['SIGN_COMMAND'] }}: - MAKECAT: $(MAKECAT) - SIGN_COMMAND: $(SIGN_COMMAND) - ${{ each alg in parameters.HashAlgorithms }}: - powershell: | From faec740f0d524f4f67fb3278a8d525fe6cdd3203 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 23:06:32 +0100 Subject: [PATCH 13/16] Properly name index and improve upload code --- windows-release/merge-and-upload.py | 71 +++++++++++++++-------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py index 62536b03..0565748f 100644 --- a/windows-release/merge-and-upload.py +++ b/windows-release/merge-and-upload.py @@ -331,14 +331,18 @@ def find_missing_from_index(url, installs): if INDEX_FILE: INDEX_FILE = Path(INDEX_FILE).absolute() - INDEX_CAT_FILE = INDEX_FILE.with_suffix(".cat") + INDEX_CAT_FILE = INDEX_FILE.with_name(f"{INDEX_FILE.name}.cat") INDEX_FILE.parent.mkdir(parents=True, exist_ok=True) with open(INDEX_FILE, "w", encoding="utf-8") as f: json.dump(index, f) sign_json(INDEX_CAT_FILE, INDEX_FILE) + INDEX_CAT_URL = f"{INDEX_URL}.cat" + INDEX_CAT_PATH = f"{INDEX_PATH}.cat" else: INDEX_CAT_FILE = None + INDEX_CAT_URL = None + INDEX_CAT_PATH = None if MANIFEST_FILE: @@ -376,37 +380,36 @@ def find_missing_from_index(url, installs): print("Expecting mtime", INDEX_MTIME, "but saw", mtime) sys.exit(1) +TO_PURGE = [i["url"] for i, *_ in UPLOADS] -if not NO_UPLOAD: - if MANIFEST_FILE: - print("Uploading", MANIFEST_FILE, "to", MANIFEST_URL) - upload_ssh(MANIFEST_FILE, MANIFEST_PATH) - - if INDEX_FILE: - print("Uploading", INDEX_FILE, "to", INDEX_URL) - upload_ssh(INDEX_FILE, INDEX_PATH) - - if INDEX_CAT_FILE: - print("Uploading", INDEX_CAT_FILE, "to", f"{INDEX_URL}.cat") - upload_ssh(INDEX_CAT_FILE, f"{INDEX_PATH}.cat") - - print("Purging", len(UPLOADS), "uploaded files") - parents = set() - for i, *_ in UPLOADS: - purge(i["url"]) - parents.add(i["url"].rpartition("/")[0] + "/") - for i in parents: - purge(i) - if MANIFEST_URL: - purge(MANIFEST_URL) - purge(MANIFEST_URL.rpartition("/")[0] + "/") - if INDEX_URL: - purge(INDEX_URL) - purge(INDEX_URL.rpartition("/")[0] + "/") - missing = find_missing_from_index(INDEX_URL, [i for i, *_ in UPLOADS]) - if missing: - print("##[error]Lost a race with another publish step!") - print("Index at", INDEX_URL, "does not contain installs:") - for m in missing: - print(m["id"], m["sort-version"]) - sys.exit(1) +if MANIFEST_FILE: + print("Uploading", MANIFEST_FILE, "to", MANIFEST_URL) + upload_ssh(MANIFEST_FILE, MANIFEST_PATH) + TO_PURGE.append(MANIFEST_URL) + +if INDEX_FILE: + print("Uploading", INDEX_FILE, "to", INDEX_URL) + upload_ssh(INDEX_FILE, INDEX_PATH) + TO_PURGE.append(INDEX_URL) + +if INDEX_CAT_FILE: + print("Uploading", INDEX_CAT_FILE, "to", INDEX_CAT_URL) + upload_ssh(INDEX_CAT_FILE, INDEX_CAT_PATH) + TO_PURGE.append(INDEX_CAT_URL) + +# Calculate directory parents for all files +TO_PURGE.extend({i.rpartition("/")[0] + "/" for i in TO_PURGE}) + +print("Purging", len(TO_PURGE), "uploaded files, indexes and directories") + +for i in TO_PURGE: + purge(i) + +if INDEX_URL: + missing = find_missing_from_index(INDEX_URL, [i for i, *_ in UPLOADS]) + if missing: + print("##[error]Lost a race with another publish step!") + print("Index at", INDEX_URL, "does not contain installs:") + for m in missing: + print(m["id"], m["sort-version"]) + sys.exit(1) From ca13434ec16aa9185ead0ca5e9ea956e7485421a Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 23:17:37 +0100 Subject: [PATCH 14/16] Add description to legacy signing --- windows-release/sign-files.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index bf41b5de..7661e0ea 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -51,9 +51,9 @@ steps: Write-Host "##vso[task.setvariable variable=MAKECAT]$(gi signtool\x64\makecat.exe)" ConvertTo-Json @{ - Endpoint=$env:TSE; - CodeSigningAccountName=$env:TSA; - CertificateProfileName=$env:TSCP; + Endpoint=$env:ASE; + CodeSigningAccountName=$env:ASA; + CertificateProfileName=$env:ASCP; # Only allow Azure CLI credentials ExcludeCredentials=@( "ManagedIdentityCredential", @@ -70,7 +70,8 @@ steps: Write-Host "##vso[task.setvariable variable=SIGNTOOL_METADATA]$(gi signtool\metadata.json)" $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td sha256 ' + ` - "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)""" + "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)"" " + ` + "/d ""${env:DESCRIPTION}""" Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" @@ -81,9 +82,10 @@ steps: workingDirectory: $(Pipeline.Workspace) displayName: 'Download signtool binaries' env: - TSE: $(TrustedSigningUri) - TSA: $(TrustedSigningAccount) - TSCP: $(TrustedSigningCertificateName) + ASE: $(TrustedSigningUri) + ASA: $(TrustedSigningAccount) + ASCP: $(TrustedSigningCertificateName) + DESCRIPTION: $(SigningDescription) EXPORT_COMMAND: ${{ parameters.ExportLegacyCommand }} - ${{ if parameters.AzureServiceConnectionName }}: From c15173f4940da0a31c17bfe1f35a4cb3c6ceec38 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 23:29:20 +0100 Subject: [PATCH 15/16] Ensure SigningDescription is always set --- windows-release/azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/windows-release/azure-pipelines.yml b/windows-release/azure-pipelines.yml index 55d99987..c4460195 100644 --- a/windows-release/azure-pipelines.yml +++ b/windows-release/azure-pipelines.yml @@ -146,6 +146,8 @@ variables: IsRealSigned: false ${{ if ne(parameters.SigningDescription, '(default)') }}: SigningDescription: ${{ parameters.SigningDescription }} + ${{ else }}: + SigningDescription: '' PublishARM64: ${{ parameters.DoARM64 }} # QUEUE TIME VARIABLES # OverrideNugetVersion: '' From 79fab191c0601509ad612b65b994ed8e9edf64ca Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 13 Apr 2026 23:35:47 +0100 Subject: [PATCH 16/16] Set signing description for feed --- windows-release/stage-publish-pymanager.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml index 46a0fde8..f951b95f 100644 --- a/windows-release/stage-publish-pymanager.yml +++ b/windows-release/stage-publish-pymanager.yml @@ -36,6 +36,10 @@ jobs: - group: CPythonSign - ${{ if eq(parameters.SigningCertificate, 'TestSign') }}: - group: CPythonTestSign + # Override the SigningDescription here, since we're only signing the feed + # and not the actual binaries. + - name: SigningDescription + value: "Python $(Build.BuildNumber)" workspace: clean: all