diff --git a/windows-release/azure-pipelines.yml b/windows-release/azure-pipelines.yml index f4e08765..c4460195 100644 --- a/windows-release/azure-pipelines.yml +++ b/windows-release/azure-pipelines.yml @@ -146,6 +146,8 @@ variables: IsRealSigned: false ${{ if ne(parameters.SigningDescription, '(default)') }}: SigningDescription: ${{ parameters.SigningDescription }} + ${{ else }}: + SigningDescription: '' PublishARM64: ${{ parameters.DoARM64 }} # QUEUE TIME VARIABLES # OverrideNugetVersion: '' @@ -320,6 +322,7 @@ stages: BuildToPublish: ${{ parameters.BuildToPublish }} DoEmbed: ${{ parameters.DoEmbed }} DoFreethreaded: ${{ parameters.DoFreethreaded }} + SigningCertificate: ${{ iif(eq(parameters.SigningCertificate, 'Unsigned'), '', parameters.SigningCertificate) }} - ${{ if eq(parameters.DoMSI, 'true') }}: - template: stage-publish-pythonorg.yml parameters: diff --git a/windows-release/merge-and-upload.py b/windows-release/merge-and-upload.py index 87bbaf6a..0565748f 100644 --- a/windows-release/merge-and-upload.py +++ b/windows-release/merge-and-upload.py @@ -20,12 +20,18 @@ UPLOAD_USER = os.getenv("UPLOAD_USER", "") NO_UPLOAD = os.getenv("NO_UPLOAD", "no")[:1].lower() in "yt1" LOCAL_INDEX = os.getenv("LOCAL_INDEX", "no")[:1].lower() in "yt1" +SIGN_COMMAND = os.getenv("SIGN_COMMAND", "") def find_cmd(env, exe): cmd = os.getenv(env) if cmd: - return Path(cmd) + cmd = Path(cmd) + if not cmd.is_file(): + raise RuntimeError( + f"Could not find {cmd} to perform upload. Incorrect %{env}% setting." + ) + return cmd for p in os.getenv("PATH", "").split(";"): if p: cmd = Path(p) / exe @@ -40,6 +46,7 @@ def find_cmd(env, exe): PLINK = find_cmd("PLINK", "plink.exe") PSCP = find_cmd("PSCP", "pscp.exe") +MAKECAT = find_cmd("MAKECAT", "makecat.exe") def _std_args(cmd): @@ -60,7 +67,9 @@ class RunError(Exception): pass -def _run(*args): +def _run(*args, single_cmd=False): + if single_cmd: + args = args[0] with subprocess.Popen( args, stdout=subprocess.PIPE, @@ -193,6 +202,43 @@ def calculate_uploads(): ) +def sign_json(cat_file, *files): + if not MAKECAT: + if not UPLOAD_HOST or NO_UPLOAD: + print("makecat.exe not found, but not uploading, so skip signing.") + return + raise RuntimeError("No makecat.exe found") + if not SIGN_COMMAND: + if not UPLOAD_HOST or NO_UPLOAD: + print("No signing command set, but not uploading, so skip signing.") + return + raise RuntimeError("No SIGN_COMMAND set") + + cat = Path(cat_file).absolute() + cdf = cat.with_suffix(".cdf") + cdf.parent.mkdir(parents=True, exist_ok=True) + + with open(cdf, "w", encoding="ansi") as f: + print("[CatalogHeader]", file=f) + print("Name=", cat.name, sep="", file=f) + print("ResultDir=", cat.parent, sep="", file=f) + print("PublicVersion=0x00000001", file=f) + print("CatalogVersion=2", file=f) + print("HashAlgorithms=SHA256", file=f) + print("EncodingType=", file=f) + print(file=f) + print("[CatalogFiles]", file=f) + for a in map(Path, files): + print("", a.name, "=", a.absolute(), sep="", file=f) + + _run(MAKECAT, "-v", cdf) + if not cat.is_file(): + raise FileNotFoundError(cat) + # Pass as a single arg because the command variable has its own arguments + _run(f'{SIGN_COMMAND} "{cat}"', single_cmd=True) + cdf.unlink() + + def remove_and_insert(index, new_installs): new = {(i["id"].casefold(), i["sort-version"].casefold()) for i in new_installs} to_remove = [ @@ -274,6 +320,7 @@ def find_missing_from_index(url, installs): except FileNotFoundError: pass + print(INDEX_PATH, "mtime =", INDEX_MTIME) @@ -284,10 +331,20 @@ def find_missing_from_index(url, installs): if INDEX_FILE: INDEX_FILE = Path(INDEX_FILE).absolute() + INDEX_CAT_FILE = INDEX_FILE.with_name(f"{INDEX_FILE.name}.cat") INDEX_FILE.parent.mkdir(parents=True, exist_ok=True) with open(INDEX_FILE, "w", encoding="utf-8") as f: json.dump(index, f) + sign_json(INDEX_CAT_FILE, INDEX_FILE) + INDEX_CAT_URL = f"{INDEX_URL}.cat" + INDEX_CAT_PATH = f"{INDEX_PATH}.cat" +else: + INDEX_CAT_FILE = None + INDEX_CAT_URL = None + INDEX_CAT_PATH = None + + if MANIFEST_FILE: # Use the sort-version so that the manifest name includes prerelease marks MANIFEST_FILE = Path(MANIFEST_FILE).absolute() @@ -323,33 +380,36 @@ def find_missing_from_index(url, installs): print("Expecting mtime", INDEX_MTIME, "but saw", mtime) sys.exit(1) +TO_PURGE = [i["url"] for i, *_ in UPLOADS] + +if MANIFEST_FILE: + print("Uploading", MANIFEST_FILE, "to", MANIFEST_URL) + upload_ssh(MANIFEST_FILE, MANIFEST_PATH) + TO_PURGE.append(MANIFEST_URL) -if not NO_UPLOAD: - if MANIFEST_FILE: - print("Uploading", MANIFEST_FILE, "to", MANIFEST_URL) - upload_ssh(MANIFEST_FILE, MANIFEST_PATH) - - if INDEX_FILE: - print("Uploading", INDEX_FILE, "to", INDEX_URL) - upload_ssh(INDEX_FILE, INDEX_PATH) - - print("Purging", len(UPLOADS), "uploaded files") - parents = set() - for i, *_ in UPLOADS: - purge(i["url"]) - parents.add(i["url"].rpartition("/")[0] + "/") - for i in parents: - purge(i) - if MANIFEST_URL: - purge(MANIFEST_URL) - purge(MANIFEST_URL.rpartition("/")[0] + "/") - if INDEX_URL: - purge(INDEX_URL) - purge(INDEX_URL.rpartition("/")[0] + "/") - missing = find_missing_from_index(INDEX_URL, [i for i, *_ in UPLOADS]) - if missing: - print("##[error]Lost a race with another publish step!") - print("Index at", INDEX_URL, "does not contain installs:") - for m in missing: - print(m["id"], m["sort-version"]) - sys.exit(1) +if INDEX_FILE: + print("Uploading", INDEX_FILE, "to", INDEX_URL) + upload_ssh(INDEX_FILE, INDEX_PATH) + TO_PURGE.append(INDEX_URL) + +if INDEX_CAT_FILE: + print("Uploading", INDEX_CAT_FILE, "to", INDEX_CAT_URL) + upload_ssh(INDEX_CAT_FILE, INDEX_CAT_PATH) + TO_PURGE.append(INDEX_CAT_URL) + +# Calculate directory parents for all files +TO_PURGE.extend({i.rpartition("/")[0] + "/" for i in TO_PURGE}) + +print("Purging", len(TO_PURGE), "uploaded files, indexes and directories") + +for i in TO_PURGE: + purge(i) + +if INDEX_URL: + missing = find_missing_from_index(INDEX_URL, [i for i, *_ in UPLOADS]) + if missing: + print("##[error]Lost a race with another publish step!") + print("Index at", INDEX_URL, "does not contain installs:") + for m in missing: + print(m["id"], m["sort-version"]) + sys.exit(1) diff --git a/windows-release/sign-files.yml b/windows-release/sign-files.yml index 6809d9c9..7661e0ea 100644 --- a/windows-release/sign-files.yml +++ b/windows-release/sign-files.yml @@ -6,33 +6,87 @@ parameters: ExtractDir: '' SigningCertificate: '' ExportCommand: '' + ExportLegacyCommand: '' ContinueOnError: false + InstallTool: true + InstallLegacyTool: false AzureServiceConnectionName: 'Python Signing' steps: -- ${{ if parameters.SigningCertificate }}: - - powershell: | - # Install sign tool - dotnet tool install --global --prerelease sign - $signtool = (gcm sign -EA SilentlyContinue).Source - if (-not $signtool) { - $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName - } - $signargs = 'code trusted-signing -v Information ' + ` - '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + ` - '-tse "$(TrustedSigningUri)" -tsa "$(TrustedSigningAccount)" -tscp "$(TrustedSigningCertificateName)" ' + ` - '-d "$(SigningDescription)" ' +- ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}: + - ${{ if eq(parameters.InstallTool, 'true') }}: + - powershell: | + # Install sign tool + dotnet tool install --global --prerelease sign + $signtool = (gcm sign -EA SilentlyContinue).Source + if (-not $signtool) { + $signtool = (gi "${env:USERPROFILE}\.dotnet\tools\sign.exe").FullName + } + $signargs = 'code artifact-signing -v Information ' + ` + '-fd sha256 -t http://timestamp.acs.microsoft.com -td sha256 ' + ` + "-ase ""${env:ASE}"" -asa ""${env:ASA}"" -ascp ""${env:ASCP}"" " + ` + "-act azure-cli -d ""${env:DESCRIPTION}""" + + Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" + Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" + if ($env:EXPORT_COMMAND) { + $signcmd = """$signtool"" $signargs" + Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" + } + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Install Azure Artifact Signing tools' + env: + ASE: $(TrustedSigningUri) + ASA: $(TrustedSigningAccount) + ASCP: $(TrustedSigningCertificateName) + DESCRIPTION: $(SigningDescription) + EXPORT_COMMAND: ${{ parameters.ExportCommand }} + + - ${{ if eq(parameters.InstallLegacyTool, 'true') }}: + - powershell: | + git clone https://github.com/python/cpython-bin-deps --revision fb06137dccc43ed5b030cdd9e3560990b37f39da --depth 1 --progress -v "signtool" + + $signtool = gi signtool\x64\signtool.exe + $dlib = gi signtool\azure_trusted_signing\x64\Azure.CodeSigning.Dlib.dll + Write-Host "##vso[task.setvariable variable=MAKECAT]$(gi signtool\x64\makecat.exe)" + + ConvertTo-Json @{ + Endpoint=$env:ASE; + CodeSigningAccountName=$env:ASA; + CertificateProfileName=$env:ASCP; + # Only allow Azure CLI credentials + ExcludeCredentials=@( + "ManagedIdentityCredential", + "WorkloadIdentityCredential", + "SharedTokenCacheCredential", + "EnvironmentCredential", + "VisualStudioCredential", + "VisualStudioCodeCredential", + "AzurePowerShellCredential", + "AzureDeveloperCliCredential", + "InteractiveBrowserCredential" + ); + } | Out-File signtool\metadata.json -Encoding ascii + Write-Host "##vso[task.setvariable variable=SIGNTOOL_METADATA]$(gi signtool\metadata.json)" - Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" - Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" - if ($env:EXPORT_COMMAND) { - $signcmd = """$signtool"" $signargs" - Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" - } - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Install Trusted Signing tools' - env: - EXPORT_COMMAND: ${{ parameters.ExportCommand }} + $signargs = 'sign /v /fd sha256 /tr http://timestamp.acs.microsoft.com /td sha256 ' + ` + "/dlib ""$dlib"" /dmdf ""$(gi signtool\metadata.json)"" " + ` + "/d ""${env:DESCRIPTION}""" + Write-Host "##vso[task.setvariable variable=__TrustedSigningCmd]$signtool" + Write-Host "##vso[task.setvariable variable=__TrustedSigningArgs]$signargs" + + if ($env:EXPORT_COMMAND) { + $signcmd = """$signtool"" $signargs" + Write-Host "##vso[task.setvariable variable=${env:EXPORT_COMMAND}]$signcmd" + } + workingDirectory: $(Pipeline.Workspace) + displayName: 'Download signtool binaries' + env: + ASE: $(TrustedSigningUri) + ASA: $(TrustedSigningAccount) + ASCP: $(TrustedSigningCertificateName) + DESCRIPTION: $(SigningDescription) + EXPORT_COMMAND: ${{ parameters.ExportLegacyCommand }} - ${{ if parameters.AzureServiceConnectionName }}: # We sign in once with the AzureCLI task, as it uses OIDC to obtain a @@ -45,28 +99,28 @@ steps: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - "##vso[task.setvariable variable=AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}" - "##vso[task.setvariable variable=AZURE_ID_TOKEN;issecret=true]${env:idToken}" - "##vso[task.setvariable variable=AZURE_TENANT_ID;issecret=true]${env:tenantId}" + "##vso[task.setvariable variable=__AZURE_CLIENT_ID;issecret=true]${env:servicePrincipalId}" + "##vso[task.setvariable variable=__AZURE_ID_TOKEN;issecret=true]${env:idToken}" + "##vso[task.setvariable variable=__AZURE_TENANT_ID;issecret=true]${env:tenantId}" addSpnToEnvironment: true - powershell: > az login --service-principal - -u $(AZURE_CLIENT_ID) - --tenant $(AZURE_TENANT_ID) + -u $(__AZURE_CLIENT_ID) + --tenant $(__AZURE_TENANT_ID) --allow-no-subscriptions - --federated-token $(AZURE_ID_TOKEN) + --federated-token $(__AZURE_ID_TOKEN) displayName: 'Authenticate signing tools (2/2)' - ${{ if parameters.Include }}: - powershell: | - if ("${{ parameters.Exclude }}") { - $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File) + if ($env:EXCLUDE) { + $files = (dir ($env:INCLUDE -split ',').Trim() -Exclude ($env:EXCLUDE -split ',').Trim() -File) } else { - $files = (dir ${{ parameters.Include }} -File) + $files = (dir ($env:INCLUDE -split ',').Trim() -File) } if ($env:FILTER) { - ($env:FILTER -split ';') -join "`n" | Out-File __filelist.txt -Encoding utf8 + ($env:FILTER -split ';').Trim() -join "`n" | Out-File __filelist.txt -Encoding utf8 } else { "*" | Out-File __filelist.txt -Encoding utf8 } @@ -82,31 +136,37 @@ steps: continueOnError: true workingDirectory: ${{ parameters.WorkingDir }} env: + INCLUDE: ${{ parameters.Include }} + EXCLUDE: ${{ parameters.Exclude }} TRUSTED_SIGNING_CMD: $(__TrustedSigningCmd) TRUSTED_SIGNING_ARGS: $(__TrustedSigningArgs) ${{ if parameters.Filter }}: FILTER: ${{ parameters.Filter }} -- ${{ if parameters.ExtractDir }}: - - powershell: | - if ("${{ parameters.Exclude }}") { - $files = (dir ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }} -File) - } else { - $files = (dir ${{ parameters.Include }} -File) - } - $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1 - if (-not $c) { - Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}" - exit - } + - ${{ if parameters.ExtractDir }}: + - powershell: | + if ($env:EXCLUDE) { + $files = (dir ($env:INCLUDE -split ',').Trim() -Exclude ($env:EXCLUDE -split ',').Trim() -File) + } else { + $files = (dir ($env:INCLUDE -split ',').Trim() -File) + } + $c = $files | %{ (Get-AuthenticodeSignature $_).SignerCertificate } | ?{ $_ -ne $null } | select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for ${{ parameters.SigningCertificate }}" + exit + } - $d = mkdir "${{ parameters.ExtractDir }}" -Force - $cf = "$d\cert.cer" - [IO.File]::WriteAllBytes($cf, $c.RawData) - $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower() + $d = mkdir $env:EXTRACT_DIR -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.RawData) + $csha = (Get-FileHash $cf -Algorithm SHA256).Hash.ToLower() - $info = @{ Subject=$c.Subject; SHA256=$csha; } - $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" - displayName: "Extract certificate info" - workingDirectory: ${{ parameters.WorkingDir }} + $info = @{ Subject=$c.Subject; SHA256=$csha; } + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + workingDirectory: ${{ parameters.WorkingDir }} + env: + INCLUDE: ${{ parameters.Include }} + EXCLUDE: ${{ parameters.Exclude }} + EXTRACT_DIR: ${{ parameters.ExtractDir }} diff --git a/windows-release/stage-publish-pymanager.yml b/windows-release/stage-publish-pymanager.yml index f24f5073..f951b95f 100644 --- a/windows-release/stage-publish-pymanager.yml +++ b/windows-release/stage-publish-pymanager.yml @@ -3,6 +3,7 @@ parameters: DoFreethreaded: false DoEmbed: false HashAlgorithms: ['SHA256'] + SigningCertificate: '' Artifacts: - name: win32 @@ -31,6 +32,14 @@ jobs: variables: - group: PythonOrgPublish + - ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: + - group: CPythonSign + - ${{ if eq(parameters.SigningCertificate, 'TestSign') }}: + - group: CPythonTestSign + # Override the SigningDescription here, since we're only signing the feed + # and not the actual binaries. + - name: SigningDescription + value: "Python $(Build.BuildNumber)" workspace: clean: all @@ -57,18 +66,29 @@ jobs: artifact: pymanager_${{ a.name }} targetPath: $(Build.BinariesDirectory)\${{ a.name }} - - task: DownloadSecureFile@1 - name: sshkey - inputs: - secureFile: pydotorg-ssh.ppk - displayName: 'Download PuTTY key' + - ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: + - task: DownloadSecureFile@1 + name: sshkey + inputs: + secureFile: pydotorg-ssh.ppk + displayName: 'Download PuTTY key' - powershell: | - git clone https://github.com/python/cpython-bin-deps --branch putty --single-branch --depth 1 --progress -v "putty" + git clone https://github.com/python/cpython-bin-deps --revision 9f9e6fc31a55406ee5ff0198ea47bbb445eeb942 --depth 1 --progress -v "putty" "##vso[task.prependpath]$(gi putty)" workingDirectory: $(Pipeline.Workspace) displayName: 'Download PuTTY binaries' + # Use the template just to configure the signing tool. + # This will set MAKECAT and SIGN_COMMAND to be injected into later build steps + - template: sign-files.yml + parameters: + Include: "" + InstallTool: false + InstallLegacyTool: true + ExportLegacyCommand: SIGN_COMMAND + SigningCertificate: ${{ parameters.SigningCertificate }} + - powershell: | if ($env:FILENAME) { "##vso[task.setvariable variable=_PyManagerIndexFilename]${env:FILENAME}" @@ -81,38 +101,38 @@ jobs: FILENAME: $(PyManagerIndexFilename) displayName: 'Infer index filename' - - powershell: | - "Uploading following packages:" - (dir "__install__.*.json").FullName - (dir "*\__install__.json").FullName - python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" - workingDirectory: $(Build.BinariesDirectory) - condition: and(succeeded(), ne(variables['IsRealSigned'], 'true')) - displayName: 'Produce uploadable ZIPs' - env: - NO_UPLOAD: 1 - INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' - INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' - MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' + - ${{ if ne(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}: + - powershell: | + "Preparing following packages:" + (dir "__install__.*.json").FullName + (dir "*\__install__.json").FullName + python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Produce uploadable ZIPs (no upload)' + env: + NO_UPLOAD: 1 + INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' + INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' + MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' - - powershell: | - "Uploading following packages:" - (dir "__install__.*.json").FullName - (dir "*\__install__.json").FullName - python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" - workingDirectory: $(Build.BinariesDirectory) - condition: and(succeeded(), eq(variables['IsRealSigned'], 'true')) - displayName: 'Upload ZIPs' - env: - INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' - INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' - MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' - UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix) - UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix) - UPLOAD_HOST: $(PyDotOrgServer) - UPLOAD_HOST_KEY: $(PyDotOrgHostKey) - UPLOAD_USER: $(PyDotOrgUsername) - UPLOAD_KEYFILE: $(sshkey.secureFilePath) + - ${{ else }}: + - powershell: | + "Uploading following packages:" + (dir "__install__.*.json").FullName + (dir "*\__install__.json").FullName + python "$(Build.SourcesDirectory)\windows-release\merge-and-upload.py" + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Upload ZIPs' + env: + INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)' + INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)' + MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json' + UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix) + UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix) + UPLOAD_HOST: $(PyDotOrgServer) + UPLOAD_HOST_KEY: $(PyDotOrgHostKey) + UPLOAD_USER: $(PyDotOrgUsername) + UPLOAD_KEYFILE: $(sshkey.secureFilePath) - ${{ each alg in parameters.HashAlgorithms }}: - powershell: |