From 06b14e2d3cce913de70deb7234a7562e90225fd8 Mon Sep 17 00:00:00 2001 From: Malcolm Daigle Date: Thu, 27 Aug 2026 13:44:51 -0700 Subject: [PATCH 01/51] Use Windows auth for local perf runs Avoid SQL Server 2025 PBKDF2 login overhead for local Windows benchmarks while preserving SQL authentication for external endpoints. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/perf/README.md | 17 +++++ eng/pipelines/perf/scripts/run-perf-tests.ps1 | 69 +++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/perf/README.md b/eng/pipelines/perf/README.md index be925f5431..2ea85fa31c 100644 --- a/eng/pipelines/perf/README.md +++ b/eng/pipelines/perf/README.md @@ -216,6 +216,23 @@ supplies the isolated dedicated host, the tuned SQL instance, and the disjoint c | Interleaving | In `interleaved` mode the harness runs **one benchmark unit at a time, baseline then candidate back-to-back**, so both sides see the same host state (see below). | | Best-of-N confirmation | A unit flagged in the first interleaved pass is re-run `confirmationRuns` times; a regression is **confirmed** only on a strict majority. Unconfirmed flags are reported but never fail the gate. | +### Windows physical opens + +The Windows connection probe measured raw TCP at about 0.06 ms but a physical +`SqlConnection.Open()` at about 158 ms when the runner used the `sa` SQL login, with the same result +from native and managed SNI. Changing TLS and TCP acknowledgment settings did not change that +latency. The Windows image runs SQL Server 2025, which verifies SQL-login passwords with 100,000 +PBKDF2 iterations; Microsoft documents the resulting login-performance impact. + +The benchmark process runs through a public-key SSH token, which cannot delegate Windows credentials +to the VM's private network address. When the injected SQL endpoint resolves to a local interface, +the Windows harness connects to the same SQL Server 2025 instance over local TCP loopback with +integrated authentication. It grants the ephemeral VM identity `db_owner` in the perf database; `sa` +remains unchanged and is used only for one-time setup. External SQL endpoints continue to use the +injected address and SQL authentication. This stays on SQL Server 2025's supported authentication +path while preventing password hashing from dominating local tests that intentionally create +physical connections. + ### Interleaving + best-of-N (run model) `benchmarkRunMode` selects how the two variants are measured: diff --git a/eng/pipelines/perf/scripts/run-perf-tests.ps1 b/eng/pipelines/perf/scripts/run-perf-tests.ps1 index c91239eb80..e40a228dc0 100644 --- a/eng/pipelines/perf/scripts/run-perf-tests.ps1 +++ b/eng/pipelines/perf/scripts/run-perf-tests.ps1 @@ -239,6 +239,67 @@ if ($sqlcmd) { throw "sqlcmd was not found on the VM; cannot create the perf database [$DbName]." } +# SQL Server 2025 verifies SQL-authentication passwords with 100,000 PBKDF2 iterations. That work +# costs about 150 ms per physical login and overwhelms connection benchmarks that intentionally +# disable or clear pooling. Use the supported Windows-authentication path over local TCP loopback. +function Test-IsLocalSqlServer { + param([string] $Server) + + try { + $serverAddresses = [System.Net.Dns]::GetHostAddresses($Server) + } catch [System.Net.Sockets.SocketException] { + Write-Warning "Could not resolve SQL Server [$Server] while checking whether it is local." + return $false + } + + $localAddresses = [System.Net.NetworkInformation.NetworkInterface]::GetAllNetworkInterfaces() | + ForEach-Object { $_.GetIPProperties().UnicastAddresses } | + ForEach-Object { $_.Address } + return $null -ne ($serverAddresses | Where-Object { $localAddresses -contains $_ } | + Select-Object -First 1) +} + +if (Test-IsLocalSqlServer $SqlServer) { + $BenchmarkSqlServer = "localhost" + $benchmarkIdentity = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name + $identityLiteral = $benchmarkIdentity.Replace("'", "''") + $configureBenchmarkIdentity = @" +DECLARE @ddl nvarchar(max); +IF SUSER_ID(N'$identityLiteral') IS NULL +BEGIN + SET @ddl = N'CREATE LOGIN ' + QUOTENAME(N'$identityLiteral') + N' FROM WINDOWS'; + EXEC sys.sp_executesql @ddl; +END +USE [$DbName]; +IF USER_ID(N'$identityLiteral') IS NULL +BEGIN + SET @ddl = N'CREATE USER ' + QUOTENAME(N'$identityLiteral') + + N' FOR LOGIN ' + QUOTENAME(N'$identityLiteral'); + EXEC sys.sp_executesql @ddl; +END; +IF IS_ROLEMEMBER(N'db_owner', N'$identityLiteral') <> 1 +BEGIN + SET @ddl = N'ALTER ROLE [db_owner] ADD MEMBER ' + QUOTENAME(N'$identityLiteral'); + EXEC sys.sp_executesql @ddl; +END; +"@ + Invoke-Native { + & $sqlcmd.Source -S $SqlServer -U sa -P $SqlPassword -C -b -l 30 -Q $configureBenchmarkIdentity + } "sqlcmd failed to configure Windows benchmark identity [$benchmarkIdentity]" + Invoke-Native { + & $sqlcmd.Source -S "tcp:$BenchmarkSqlServer,1433" -E -d $DbName -C -b -l 15 ` + -Q "SET NOCOUNT ON; SELECT SUSER_SNAME(), USER_NAME();" + } "Loopback integrated-authentication preflight failed for benchmark identity [$benchmarkIdentity]" + $BenchmarkConnectionString = "Server=tcp:$BenchmarkSqlServer,1433;Integrated Security=True;Initial Catalog=$DbName;TrustServerCertificate=True;Encrypt=True;" + Write-Host "Verified Windows integrated authentication over local TCP loopback." +} else { + # Preserve the injected endpoint and SQL authentication for external SQL Server deployments. + $BenchmarkSqlServer = $SqlServer + $escapedPassword = '"' + ($SqlPassword -replace '"', '""') + '"' + $BenchmarkConnectionString = "Server=tcp:$BenchmarkSqlServer,1433;User ID=sa;Password=$escapedPassword;Initial Catalog=$DbName;TrustServerCertificate=True;Encrypt=True;" + Write-Host "Using SQL authentication for external SQL Server [$BenchmarkSqlServer]." +} + #################################################################################################### # Noise-reduction controls (InternalDriverTools wiki 339, "Reducing Noise in Performance Tests"). # @@ -308,11 +369,7 @@ $rawConfig = Get-Content $srcConfig -Raw $rawConfig = ($rawConfig -split "`n" | ForEach-Object { $_ -replace '(?m)^\s*//.*$', '' }) -join "`n" $cfg = ConvertFrom-Json $rawConfig -# SqlClient connection-string values may be wrapped in double quotes; doubling any embedded double -# quote lets a password containing ';', '=', spaces or single quotes be parsed as a single literal -# value instead of corrupting the connection string. -$escapedPassword = '"' + ($SqlPassword -replace '"', '""') + '"' -$cfg.ConnectionString = "Server=tcp:$SqlServer,1433;User ID=sa;Password=$escapedPassword;Initial Catalog=$DbName;TrustServerCertificate=True;Encrypt=False;" +$cfg.ConnectionString = $BenchmarkConnectionString # Apply the optional SqlClient behaviour overrides supplied by the pipeline. An empty value leaves # the checked-in default untouched; otherwise the flag is forced to the requested boolean so the # benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. @@ -328,7 +385,7 @@ Set-CfgBool $cfg "UseManagedSniOnWindows" $UseManagedSniOnWindows Set-CfgBool $cfg "UseOptimizedAsyncBehaviour" $UseOptimizedAsyncBehaviour Set-CfgBool $cfg "UseConnectionPoolV2" $UseConnectionPoolV2 $cfg | ConvertTo-Json -Depth 10 | Set-Content -Path $RunnerConfig -Encoding UTF8 -Write-Host "Wrote runner config to $RunnerConfig (Server=tcp:$SqlServer,1433; Initial Catalog=$DbName)" +Write-Host "Wrote runner config to $RunnerConfig (Server=tcp:$BenchmarkSqlServer,1433; Initial Catalog=$DbName)" #################################################################################################### # 4 & 5. Run the benchmarks, pinned to the reserved client CPU set. From 9449c98b10a30488c7cb8a6e24ddd2eb43e74558 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:33:44 -0700 Subject: [PATCH 02/51] [Scheduled Run] Localized resource files from OneLocBuild (#4607) Co-authored-by: SqlClient DevOps --- src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx | 4 ++-- src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx | 4 ++-- .../src/Resources/Strings.zh-Hans.resx | 4 ++-- .../src/Resources/Strings.zh-Hant.resx | 4 ++-- 13 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx index 4e20d68eeb..f85eda7d30 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx @@ -2137,10 +2137,10 @@ Certifikát poskytnutý serverem neodpovídá certifikátu poskytnutému možností ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Soubor certifikátu určený možností 'ServerCertificate' se nepodařilo načíst ani parsovat: '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Byla zadána možnost 'ServerCertificate', ale server nepředložil certifikát, který by s ním bylo možné porovnat. Neplatný pokus o získání třídy JsonDocument ve sloupci {0}. Třída JsonDocument se podporuje jenom pro sloupce typu json. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx index 07aa2c6b3f..7dcd133277 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx @@ -2137,10 +2137,10 @@ Das vom Server bereitgestellte Zertifikat stimmt nicht mit dem Zertifikat überein, das über die ServerCertificate-Option bereitgestellt wurde. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Die mit der Option „ServerCertificate“ angegebene Zertifikatdatei konnte nicht geladen oder geparst werden: '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Die Option „ServerCertificate“ wurde angegeben, aber der Server hat kein Zertifikat bereitgestellt, mit dem es verglichen werden kann. Ungültiger Versuch, „JsonDocument“ für Spalte „{0}“ abzurufen. „JsonDocument“ wird nur für Spalten vom Typ JSON unterstützt. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx index 5d912814b4..3471a75894 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx @@ -2137,10 +2137,10 @@ El certificado proporcionado por el servidor no coincide con el certificado proporcionado por la opción ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + No se pudo cargar ni analizar el archivo de certificado especificado por la opción "ServerCertificate": "{0}". - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Se especificó la opción 'ServerCertificate', pero el servidor no presentó un certificado que se pudiera comparar con él. Intento no válido de obtener JsonDocument en la columna '{0}'. JsonDocument solo se admite para columnas de tipo json. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx index c0082ed5d8..d73673efbb 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx @@ -2137,10 +2137,10 @@ Le certificat fourni par le serveur ne correspond pas au certificat fourni par l’option ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Le fichier de certificat spécifié par l’option « ServerCertificate » n’a pas pu être chargé ou analysé : « {0} ». - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + L’option « ServerCertificate » a été spécifiée, mais le serveur n’a présenté aucun certificat qui puisse être comparé à celui-ci. Tentative non valide d’obtention de JsonDocument sur la colonne « {0} ». JsonDocument est uniquement pris en charge pour les colonnes de type JSON. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx index 4708501a33..1e2e6764d5 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx @@ -2137,10 +2137,10 @@ Il certificato fornito dal server non corrisponde al certificato fornito dall'opzione ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Non è possibile caricare o analizzare il file del certificato specificato dall'opzione 'ServerCertificate: '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + È stata specificata l'opzione 'ServerCertificate', ma il server non ha presentato un certificato confrontabile con essa. Tentativo non valido di ottenere JsonDocument nella colonna '{0}'. JsonDocument è supportato solo per colonne di tipo JSON. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx index e6d556d6d3..6e762be388 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx @@ -2137,10 +2137,10 @@ サーバーによって提供された証明書が、ServerCertificate オプションで指定された証明書と一致しません。 - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 'ServerCertificate' オプションで指定された証明書ファイルを読み込むことも解析することもできませんでした: '{0}'。 - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 'ServerCertificate' オプションが指定されましたが、サーバーから、このオプションと照合できる証明書が提示されませんでした。 列 '{0}' で JsonDocument を取得しようとしましたが無効です。JsonDocument は json 型の列でのみサポートされています。 diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx index 688a1bee60..ccca396df1 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx @@ -2137,10 +2137,10 @@ 서버에서 제공하는 인증서가 ServerCertificate 옵션에서 제공하는 인증서와 일치하지 않습니다. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 'ServerCertificate' 옵션으로 지정된 인증서 파일을 로드하거나 구문 분석할 수 없습니다. '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 'ServerCertificate' 옵션이 지정되었지만 서버에서 비교할 수 있는 인증서를 제공하지 않았습니다. 열 '{0}'의 JsonDocument를 가져오려는 시도가 잘못되었습니다. JsonDocument는 json 형식의 열에 대해서만 지원됩니다. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx index d9656a1a8c..9a2f4e6f96 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx @@ -2137,10 +2137,10 @@ Certyfikat dostarczony przez serwer nie jest zgodny z certyfikatem dostarczonym przez opcję ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Nie można załadować lub przeanalizować pliku certyfikatu określonego przez opcję „ServerCertificate”: „{0}”. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Określono opcję „ServerCertificate”, ale serwer nie przedstawia certyfikatu, który można porównać z nim. Nieprawidłowa próba pobrania pliku JsonDocument w kolumnie „{0}”. Obiekt JsonDocument jest obsługiwany tylko w przypadku kolumn typu JSON. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx index 581068d864..2e7593f6fd 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx @@ -2137,10 +2137,10 @@ O certificado fornecido pelo servidor não corresponde ao certificado fornecido pela opção ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + O arquivo de certificado especificado pela opção ''ServerCertificate'' não pôde ser carregado ou analisado: ''{0}''. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + A opção ''ServerCertificate'' foi especificada, mas o servidor não apresentou um certificado que pudesse ser comparado com ele. Tentativa inválida de obter JsonDocument na coluna ''{0}''. Só há suporte para JsonDocument em colunas do tipo json. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx index 8b3cbff48a..17f0d5ba7e 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx @@ -2137,10 +2137,10 @@ Сертификат, предоставленный сервером, не совпадает с сертификатом, указанным в параметре ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Не удалось загрузить или проанализировать файл сертификата, указанный параметром "ServerCertificate": "{0}". - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Параметр "ServerCertificate" был указан, но сервер не представил сертификат, с которым можно было бы его сравнить. Недопустимая попытка получить JsonDocument для столбца "{0}". JsonDocument поддерживается только для столбцов типа JSON. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx index 650a21f087..f874f78401 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx @@ -2137,10 +2137,10 @@ Sunucu tarafından sağlanan sertifika, ServerCertificate seçeneği tarafından sağlanan sertifikayla eşleşmiyor. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 'ServerCertificate' seçeneğiyle belirtilen sertifika dosyası yüklenemedi veya ayrıştırılamadı: '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 'ServerCertificate' seçeneği belirtildi, ancak sunucu bununla karşılaştırılabilecek bir sertifika sunmadı. '{0}' sütununda JsonDocument alma denemesi geçersiz. JsonDocument yalnızca json türündeki sütunlar için desteklenir. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx index 77db610a7c..5a5ebe15a6 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx @@ -2137,10 +2137,10 @@ 服务器提供的证书与 ServerCertificate 选项提供的证书不匹配。 - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 无法加载或解析 "ServerCertificate" 选项指定的证书文件: "{0}"。 - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 已指定 "ServerCertificate" 选项,但服务器未提供可与其比较的证书。 在列 ‘{0}’ 上获取 JsonDocument 的尝试无效。只有 json 类型的列支持 JsonDocument。 diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx index f7d87d6908..9b0d8fd673 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx @@ -2137,10 +2137,10 @@ 伺服器提供的憑證不符合 ServerCertificate 選項所提供的憑證。 - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 無法載入或剖析 'ServerCertificate' 選項所指定的憑證檔案: '{0}'。 - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 已指定 'ServerCertificate' 選項,但伺服器未提供可與其比對的憑證。 在資料行 '{0}' 上取得 JsonDocument 的嘗試無效。JsonDocument 僅支援 JSON 類型的資料行。 From f9fb0c92598e8af8b2dabfdebdd168b6724a32f5 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:23:55 -0300 Subject: [PATCH 03/51] Make code coverage artifact names unique per attempt (#4616) --- eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml index 95c0ac0f3c..e4e28d4803 100644 --- a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml @@ -112,12 +112,13 @@ jobs: displayName: '[Debug] Show Disk Usage' # Publish the Cobertura XML coverage file as a pipeline artifact for - # debugging purposes. + # debugging purposes. Pipeline artifact names must be unique within a + # build, so include the attempt numbers to allow this job to be rerun. - task: PublishPipelineArtifact@1 displayName: Publish Cobertura XML Artifact inputs: targetPath: $(workingDir)/merge - artifact: Cobertura Merge Results + artifact: Cobertura Merge Results $(System.StageAttempt)-$(System.JobAttempt) # Publish the Cobertura reports to the pipeline to be viewed in the Azure # DevOps pipeline run UI. From 7a8d455f1208a0241401f4dd6ad4f1c644313b4b Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:14:31 -0700 Subject: [PATCH 04/51] Deprecate TransparentNetworkIPResolution (#4576) * Deprecate TransparentNetworkIPResolution Partially implements #4494 (deprecation only; no default changes). - Mark SqlConnectionStringBuilder.TransparentNetworkIPResolution obsolete in both the implementation and the reference assembly, pointing callers at MultiSubnetFailover and noting that TNIR is .NET Framework-only. - Suppress the resulting obsolete warnings at the internal call sites in SqlConnectionStringBuilder and in tests that exercise the keyword. - Document the deprecation in the SqlConnection and SqlConnectionStringBuilder doc snippets. Connection string defaults are intentionally unchanged in this version: TransparentNetworkIPResolution still defaults to true on .NET Framework and MultiSubnetFailover still defaults to false. Flipping those defaults and adding the associated compatibility switches is deferred. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Address review feedback: annotate CS0618 suppressions and use xref for ArgumentException - Add the existing '// Obsolete properties' rationale comment to the two new CS0618 suppressions around TransparentNetworkIPResolution so the intent matches the neighbouring ConnectionReset suppressions. - Use instead of a backtick literal in the SqlConnection.xml keyword table, matching the file's existing convention. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 108f631b-430f-40a3-bd20-74f19527c192 * Suppress CS0618 for TNIR usages in SqlConnectionOptionsTest (net462) The net462 legs failed to build with CS0618 because SqlConnectionOptionsTest sets SqlConnectionStringBuilder.TransparentNetworkIPResolution in two netfx-gated tests. These usages arrived on main after this branch was cut, so they were missing the suppressions applied to the other test files. Merged main into the branch so the build matches what CI compiles, and wrapped both call sites in the same '#pragma warning disable 618' pattern used elsewhere. Verified locally by building net462 with -p:TargetOs=Windows_NT for the UnitTests, FunctionalTests and ManualTests projects: all clean with 0 warnings. Confirmed the check is faithful by removing the pragmas and reproducing the exact CS0618 errors CI reported. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 108f631b-430f-40a3-bd20-74f19527c192 * Fix indentation of TNIR assignment in SqlConnectionOptionsTest The pragma-wrapped assignment lost its block-scope indentation, making it inconsistent with the matching tnirInConnString block below. The diff against main is now purely additive. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 108f631b-430f-40a3-bd20-74f19527c192 * Use xref for ArgumentException in TNIR keyword docs Matches the convention used elsewhere in SqlConnection.xml so the type reference stays linkable in generated docs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 108f631b-430f-40a3-bd20-74f19527c192 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 108f631b-430f-40a3-bd20-74f19527c192 --- doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml | 2 +- .../SqlConnectionStringBuilder.xml | 3 +++ .../ref/Microsoft.Data.SqlClient.cs | 1 + .../Data/SqlClient/SqlConnectionStringBuilder.cs | 5 +++++ .../Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs | 4 ++++ .../SimulatedServerTests/ConnectionFailoverTests.cs | 8 ++++++++ .../SimulatedServerTests/ConnectionRoutingTests.cs | 4 ++++ .../UnitTests/SimulatedServerTests/ConnectionTests.cs | 8 ++++++++ .../SimulatedServerTests/SNICloseDeadlockTest.cs | 6 ++++++ .../SNICloseHandshakeCancellationTest.cs | 2 ++ .../SimulatedServerTests/SNICloseRaceDeadlockTest.cs | 2 ++ 11 files changed, 44 insertions(+), 1 deletion(-) diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml index 05f3c774b0..5f88a1498e 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml @@ -1033,7 +1033,7 @@ The following table lists the valid names for keyword values within the
-or-

ServerSPN|N/A|The SPN for the data source. The default value is an empty string, which causes SqlClient to use the default, driver-generated SPN.

(Only available in v5.0+)| |Transaction Binding|Implicit Unbind|Controls connection association with an enlisted `System.Transactions` transaction.

Possible values are:

`Transaction Binding=Implicit Unbind;`

`Transaction Binding=Explicit Unbind;`

Implicit Unbind causes the connection to detach from the transaction when it ends. After detaching, additional requests on the connection are performed in autocommit mode. The `System.Transactions.Transaction.Current` property is not checked when executing requests while the transaction is active. After the transaction has ended, additional requests are performed in autocommit mode.

If the system ends the transaction (in the scope of a using block) before the last command completes, it will throw .

Explicit Unbind causes the connection to remain attached to the transaction until the connection is closed or an explicit `SqlConnection.TransactionEnlist(null)` is called. Beginning in .NET Framework 4.0, changes to Implicit Unbind make Explicit Unbind obsolete. An `InvalidOperationException` is thrown if `Transaction.Current` is not the enlisted transaction or if the enlisted transaction is not active.| -|Transparent Network IP Resolution

-or-

TransparentNetworkIPResolution|See description.|On .NET Framework, when the value of this key is set to `true`, the driver runs multiple connect rounds across the DNS-resolved IP addresses, with progressively larger per-attempt timeouts and a 500 ms minimum on the sequential-mode attempt, until a connection succeeds or the overall `Connect Timeout` is reached.

If the `MultiSubnetFailover` key is set to `true`, `TransparentNetworkIPResolution` is ignored.

If the `Failover Partner` key is set, `TransparentNetworkIPResolution` is ignored.

On .NET Framework, if `TransparentNetworkIPResolution` isn't specified in the connection string, the driver automatically disables TNIR when the data source is an Azure SQL endpoint (`.database.windows.net`, `.database.cloudapi.de`, `.database.usgovcloudapi.net`, `.database.chinacloudapi.cn`, or `.database.fabric.microsoft.com`), when the `Authentication` key is set to any Microsoft Entra ID method (`Active Directory Password`, `Active Directory Integrated`, `Active Directory Interactive`, `Active Directory Service Principal`, `Active Directory Device Code Flow`, `Active Directory Managed Identity`, `Active Directory MSI`, `Active Directory Default`, or `Active Directory Workload Identity`), or when the or property is set on the . For these automatic conditions, an explicit `TransparentNetworkIPResolution` value bypasses the automatic behavior: `True` enables TNIR, and `False` disables TNIR unconditionally. To restore the automatic behavior, remove the keyword from the connection string.

On .NET (Core, .NET 5+), `TransparentNetworkIPResolution` isn't a recognized connection-string keyword. Setting it (with any value) throws `ArgumentException` when the driver parses the connection string.

On .NET Framework, the value of this key must be `true`, `false`, `yes`, or `no`.

A value of `yes` is treated the same as a value of `true`.

A value of `no` is treated the same as a value of `false`.| +|Transparent Network IP Resolution

-or-

TransparentNetworkIPResolution|See description.|**Deprecated.** Use `Multi Subnet Failover` instead.

On .NET Framework, when the value of this key is set to `true`, the driver runs multiple connect rounds across the DNS-resolved IP addresses, with progressively larger per-attempt timeouts and a 500 ms minimum on the sequential-mode attempt, until a connection succeeds or the overall `Connect Timeout` is reached.

If the `MultiSubnetFailover` key is set to `true`, `TransparentNetworkIPResolution` is ignored.

If the `Failover Partner` key is set, `TransparentNetworkIPResolution` is ignored.

On .NET Framework, if `TransparentNetworkIPResolution` isn't specified in the connection string, the driver automatically disables TNIR when the data source is an Azure SQL endpoint (`.database.windows.net`, `.database.cloudapi.de`, `.database.usgovcloudapi.net`, `.database.chinacloudapi.cn`, or `.database.fabric.microsoft.com`), when the `Authentication` key is set to any Microsoft Entra ID method (`Active Directory Password`, `Active Directory Integrated`, `Active Directory Interactive`, `Active Directory Service Principal`, `Active Directory Device Code Flow`, `Active Directory Managed Identity`, `Active Directory MSI`, `Active Directory Default`, or `Active Directory Workload Identity`), or when the or property is set on the . For these automatic conditions, an explicit `TransparentNetworkIPResolution` value bypasses the automatic behavior: `True` enables TNIR, and `False` disables TNIR unconditionally. To restore the automatic behavior, remove the keyword from the connection string.

On .NET (Core, .NET 5+), `TransparentNetworkIPResolution` isn't a recognized connection-string keyword. Setting it (with any value) throws when the driver parses the connection string.

On .NET Framework, the value of this key must be `true`, `false`, `yes`, or `no`.

A value of `yes` is treated the same as a value of `true`.

A value of `no` is treated the same as a value of `false`.| |Trust Server Certificate

-or-

TrustServerCertificate|'false'|When set to `true`, TLS is used to encrypt the channel when bypassing walking the certificate chain to validate trust. If TrustServerCertificate is set to `true` and Encrypt is set to `false`, the channel is not encrypted. Recognized values are `true`, `false`, `yes`, and `no`. For more information, see [Connection String Syntax](https://learn.microsoft.com/sql/connect/ado-net/connection-string-syntax).| |Type System Version|N/A|A string value that indicates the type system the application expects. The functionality available to a client application is dependent on the version of SQL Server and the compatibility level of the database. Explicitly setting the type system version that the client application was written for avoids potential problems that could cause an application to break if a different version of SQL Server is used. **Note:** The type system version cannot be set for common language runtime (CLR) code executing in-process in SQL Server. For more information, see [SQL Server Common Language Runtime Integration](https://learn.microsoft.com/dotnet/framework/data/adonet/sql/sql-server-common-language-runtime-integration).

Possible values are:

`Type System Version=SQL Server 2012;`

`Type System Version=SQL Server 2008;`

`Type System Version=SQL Server 2005;`

`Type System Version=Latest;`

`Type System Version=SQL Server 2012;` specifies that the application will require version 11.0.0.0 of Microsoft.SqlServer.Types.dll. The other `Type System Version` settings will require version 10.0.0.0 of Microsoft.SqlServer.Types.dll.

`Latest` is obsolete and should not be used. `Latest` is equivalent to `Type System Version=SQL Server 2008;`.| |User ID

-or-

UID

-or-

User|N/A|The SQL Server login account. Not recommended. To maintain a high level of security, we strongly recommend that you use the `Integrated Security` or `Trusted_Connection` keywords instead. is a more secure way to specify credentials for a connection that uses SQL Server Authentication.

The user ID must be 128 characters or less.| diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml index 1387196677..ffeb1ad240 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml @@ -1416,6 +1416,9 @@ This property corresponds to the "ServerSPN" and "Server SPN" keys within the co A boolean value. + + This property is obsolete. Use instead. + If the Multi Subnet Failover key is set to true, Transparent Network IP Resolution is ignored. diff --git a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs index a5e3f1a473..266d0b9a1d 100644 --- a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs +++ b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs @@ -1492,6 +1492,7 @@ public override void Clear() { } #if NETFRAMEWORK /// + [System.ObsoleteAttribute("TransparentNetworkIPResolution has been deprecated and is only supported on .NET Framework. Use MultiSubnetFailover instead.")] [System.ComponentModel.DisplayNameAttribute("Transparent Network IP Resolution")] [System.ComponentModel.RefreshPropertiesAttribute(System.ComponentModel.RefreshProperties.All)] public bool TransparentNetworkIPResolution { get { throw null; } set { } } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs index 4038dbb295..bb4871ed4a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs @@ -411,7 +411,9 @@ private object GetAt(Keywords index) return ConnectionReset; #pragma warning restore 618 case Keywords.TransparentNetworkIPResolution: +#pragma warning disable 618 // Obsolete properties return TransparentNetworkIPResolution; +#pragma warning restore 618 case Keywords.NetworkLibrary: return NetworkLibrary; #endif @@ -1083,7 +1085,9 @@ public override object this[string keyword] NetworkLibrary = ConvertToString(value); break; case Keywords.TransparentNetworkIPResolution: +#pragma warning disable 618 // Obsolete properties TransparentNetworkIPResolution = ConvertToBoolean(value); +#pragma warning restore 618 break; #endif default: @@ -1879,6 +1883,7 @@ public bool ConnectionReset } /// + [Obsolete("TransparentNetworkIPResolution has been deprecated and is only supported on .NET Framework. Use MultiSubnetFailover instead.")] [DisplayName(DbConnectionStringKeywords.TransparentNetworkIpResolution)] [ResCategory(nameof(Strings.DataCategory_Source))] [ResDescription(nameof(Strings.DbConnectionString_TransparentNetworkIPResolution))] diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs index 11b4ed8637..4c15a65789 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs @@ -59,7 +59,9 @@ public void TestDefaultTnir(string dataSource, bool? tnirEnabledInConnString, bo builder.DataSource = dataSource; if (tnirEnabledInConnString.HasValue) { +#pragma warning disable 618 // TransparentNetworkIPResolution is obsolete builder.TransparentNetworkIPResolution = tnirEnabledInConnString.Value; +#pragma warning restore 618 } SqlConnectionOptions connectionString = new(builder.ConnectionString); @@ -99,7 +101,9 @@ public void TestShouldDisableTnirWithCallerSuppliedToken( SqlConnectionStringBuilder builder = new() { DataSource = dataSource }; if (tnirInConnString.HasValue) { +#pragma warning disable 618 // TransparentNetworkIPResolution is obsolete builder.TransparentNetworkIPResolution = tnirInConnString.Value; +#pragma warning restore 618 } SqlConnectionOptions connectionOptions = new(builder.ConnectionString); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs index ba1a852626..216a131ad8 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs @@ -129,7 +129,9 @@ public void NetworkError_TriggersFailover_ClearsPool() InitialCatalog = "test", MultiSubnetFailover = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; @@ -199,7 +201,9 @@ public void NetworkTimeout_ShouldFail() Encrypt = false, MultiSubnetFailover = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); @@ -246,7 +250,9 @@ public void NetworkDelay_ShouldConnectToPrimary() Pooling = false, // Disable pooling to ensure a fresh connection attempt is made MultiSubnetFailover = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); @@ -388,7 +394,9 @@ public void NetworkError_WithUserProvidedPartner_RetryEnabled_ShouldConnectToFai FailoverPartner = $"localhost,{failoverServer.EndPoint.Port}", // User provided failover partner Encrypt = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs index 262770f563..918f7228f3 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs @@ -136,7 +136,9 @@ public void NetworkDelayAtRoutedLocation_RetryDisabled_ShouldSucceed(bool multiS Encrypt = false, MultiSubnetFailover = multiSubnetFailoverEnabled, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = multiSubnetFailoverEnabled, + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); @@ -190,7 +192,9 @@ public void NetworkTimeoutAtRoutedLocation_RetryDisabled_ShouldFail() Encrypt = false, MultiSubnetFailover = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs index f797797758..1a48ccd293 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs @@ -111,7 +111,9 @@ public async Task TransientFault_RetryEnabled_ShouldSucceed_Async(uint errorCode DataSource = "localhost," + server.EndPoint.Port, Encrypt = SqlConnectionEncryptOption.Optional, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false + #pragma warning restore 618 #endif }; @@ -252,7 +254,9 @@ public async Task NetworkError_RetryEnabled_ShouldSucceed_Async(bool multiSubnet Pooling = false, // Disable pooling to ensure a fresh connection attempt is made MultiSubnetFailover = multiSubnetFailoverEnabled, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = multiSubnetFailoverEnabled + #pragma warning restore 618 #endif }; @@ -290,7 +294,9 @@ public async Task NetworkDelay_RetryDisabled_Async(bool multiSubnetFailoverEnabl Encrypt = SqlConnectionEncryptOption.Optional, MultiSubnetFailover = multiSubnetFailoverEnabled, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = multiSubnetFailoverEnabled, + #pragma warning restore 618 #endif }; @@ -338,7 +344,9 @@ public void NetworkDelay_RetryDisabled(bool multiSubnetFailoverEnabled) ConnectTimeout = 5, MultiSubnetFailover = multiSubnetFailoverEnabled, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = multiSubnetFailoverEnabled, + #pragma warning restore 618 #endif }; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseDeadlockTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseDeadlockTest.cs index 465e284b6d..fa333e9215 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseDeadlockTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseDeadlockTest.cs @@ -119,7 +119,9 @@ public void CloseOrDispose_WithPendingAsyncRead_DoesNotDeadlock(bool disposeInst // pool. Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; @@ -292,7 +294,9 @@ public void CloseOrDispose_DuringPreLoginHandshake_DoesNotDeadlock(bool disposeI ConnectRetryCount = 0, Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; @@ -502,7 +506,9 @@ public void CloseOrDispose_DuringTlsHandshake_DoesNotDeadlock(bool disposeInstea ConnectRetryCount = 0, Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseHandshakeCancellationTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseHandshakeCancellationTest.cs index 9a41c2f3c7..b11b5ed291 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseHandshakeCancellationTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseHandshakeCancellationTest.cs @@ -119,7 +119,9 @@ public void CancelOpenAsyncDuringTlsHandshake_DoesNotDeadlock() ConnectRetryCount = 0, Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseRaceDeadlockTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseRaceDeadlockTest.cs index 27ab3a7516..78a099bba4 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseRaceDeadlockTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseRaceDeadlockTest.cs @@ -106,7 +106,9 @@ public void ResponseCompletionRacesClose_DoesNotDeadlock(bool disposeInsteadOfCl // connection (reaching SNIClose) instead of returning it to the pool. Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; From 1ff4ebaf6970cc8a056574bf3bad2576208c52c5 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:15:25 -0700 Subject: [PATCH 05/51] Upgrade checkout actions to Node 24 (#4575) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/auto-assign-pr.yml | 2 +- .github/workflows/verify-aw-lock.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/auto-assign-pr.yml b/.github/workflows/auto-assign-pr.yml index ba5654969f..aab463f2e2 100644 --- a/.github/workflows/auto-assign-pr.yml +++ b/.github/workflows/auto-assign-pr.yml @@ -23,7 +23,7 @@ jobs: PR_REVIEWER_POOL: ${{ vars.PR_REVIEWER_POOL }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: sparse-checkout: | .github/scripts/auto-assign-pr.js diff --git a/.github/workflows/verify-aw-lock.yml b/.github/workflows/verify-aw-lock.yml index 1381792980..c020335650 100644 --- a/.github/workflows/verify-aw-lock.yml +++ b/.github/workflows/verify-aw-lock.yml @@ -13,7 +13,7 @@ jobs: verify: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install gh-aw extension uses: github/gh-aw-actions/setup-cli@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 From 7f82574972a400852245a6b48f1ff9eb93958581 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:35:00 -0700 Subject: [PATCH 06/51] Fix malformed UNC pipe path for IPv6 literals in managed SNI (#4558) * Reject IPv6 literal host names on the Named Pipes path in managed SNI A UNC path host component may never contain a colon, so an IPv6 literal server name composes a malformed pipe path such as \\::1\pipe\sql\query. Handing that to the OS sends the SMB redirector into an SMB session setup whose SPNEGO/NegoEx target name embeds the IPv6 literal, which can fault LSASS on Windows and force a reboot. Managed SNI defaults to TCP when no protocol prefix is given, so this is reachable only when Named Pipes is selected explicitly (np:::1) or via a UNC pipe path (\\::1\pipe\sql\query). Validate the host component in both DataSource.InferNamedPipesInformation branches, plus a final safeguard in SniProxy.CreateNpHandle, mirroring the native SNI fix. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Address review feedback: use literal ':' and document test methods - IsValidPipeHostName now compares against the literal ':' instead of the misleadingly named SemiColon constant. - Add XML summaries to every test method in DataSourceNamedPipesTests, matching the convention used by the other ManagedSni unit tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Transcribe IPv6 literals to UNC form instead of rejecting them Review feedback: rejecting IPv6 literals on the Named Pipes path blocked addresses that would otherwise work. Windows defines a UNC transcription for exactly this case (MS-DTYP 2.2.57): replace ':' with '-' and '%' with 's', then append '.ipv6-literal.net', so 2001:db8::1 becomes 2001-db8--1.ipv6-literal.net. DataSource.GetUncCompatibleHostName replaces IsValidPipeHostName and applies that transcription at both Named Pipes host-assignment sites. Colon-free hosts (host names, IPv4 literals, already-transcribed names) pass through untouched, and a colon-bearing host with no IPv6 interpretation still fails cleanly. ServerName keeps the original literal because it feeds SPN creation. CreateNpHandle keeps a colon check as a final safeguard. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Normalize ServerName for bracketed IPv6 literals Copilot review: for a bracketed IPv6 host such as np:[2001:db8::1] or \\[2001:db8::1]\pipe\sql\query, PipeHostName was transcribed but ServerName kept the brackets. ServerName feeds Dns.GetHostEntry and SPN construction in SniProxy.GetSqlServerSPNs, neither of which accepts the bracketed spelling, so lookup would fail and a malformed SPN such as MSSQLSvc/[2001:db8::1] could be produced. Factor the literal parsing into TryParseIPv6Literal, shared by the new NormalizeHostName (unwraps brackets to the canonical unbracketed form) and GetUncCompatibleHostName. Both Named Pipes host-assignment sites now normalize ServerName alongside transcribing PipeHostName. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Address IPv6 named pipe review feedback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1790b392-a88c-4aa2-a50b-90046a378db5 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1790b392-a88c-4aa2-a50b-90046a378db5 --- .../SqlClient/ManagedSni/SniProxy.netcore.cs | 138 +++++++++++- .../ManagedSni/DataSourceNamedPipesTests.cs | 212 ++++++++++++++++++ 2 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/ManagedSni/DataSourceNamedPipesTests.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs index de40956784..b5254cde75 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs @@ -275,6 +275,17 @@ private static SniNpHandle CreateNpHandle(DataSource details, TimeoutTimer timeo SniCommon.ReportSNIError(SniProviders.NP_PROV, 0, SniCommon.MultiSubnetFailoverWithNonTcpProtocol, Strings.SNI_ERROR_49); return null; } + + // Final safeguard: never hand a pipe path whose host component contains a colon to the + // OS. DataSource transcribes IPv6 literals during parsing, so anything still holding a + // colon here is malformed. See DataSource.GetUncCompatibleHostName for details. + if (string.IsNullOrEmpty(details.PipeHostName) || details.PipeHostName.IndexOf(':') != -1) + { + SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniProxy), EventType.ERR, "Invalid host name '{0}' for Named Pipes.", details.PipeHostName); + SniCommon.ReportSNIError(SniProviders.NP_PROV, 0, SniCommon.InvalidConnStringError, Strings.SNI_ERROR_25); + return null; + } + return new SniNpHandle(details.PipeHostName, details.PipeName, timeout, tlsFirst, hostNameInCertificate, serverCertificateFilename); } @@ -339,6 +350,7 @@ internal class DataSource private const string DefaultPipeName = "sql\\query"; private const string InstancePrefix = "MSSQL$"; private const string PathSeparator = "\\"; + private const string IPv6LiteralHostSuffix = ".ipv6-literal.net"; internal enum Protocol { TCP, NP, None, Admin }; @@ -632,7 +644,7 @@ private bool InferNamedPipesInformation() { // NamedPipeClientStream object will create the network path using PipeHostName and PipeName // and can be seen in its _normalizedPipePath variable in the format \\servername\pipe\MSSQL$\sql\query - PipeHostName = ServerName = tokensByBackSlash[0]; + ServerName = tokensByBackSlash[0]; PipeName = $"{InstancePrefix}{tokensByBackSlash[1]}{PathSeparator}{DefaultPipeName}"; } else @@ -643,10 +655,23 @@ private bool InferNamedPipesInformation() } else { - PipeHostName = ServerName = _dataSourceAfterTrimmingProtocol; + ServerName = _dataSourceAfterTrimmingProtocol; PipeName = SniNpHandle.DefaultPipePath; } + // An IPv6 literal must be transcribed before it can appear in a UNC pipe path, + // and ServerName must drop any brackets because it feeds DNS resolution and SPN + // construction. See GetUncCompatibleHostName and NormalizeHostName for details. + PipeHostName = GetUncCompatibleHostName(ServerName); + if (PipeHostName is null) + { + SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniProxy), EventType.ERR, "Invalid host name '{0}' for Named Pipes.", ServerName); + ReportSNIError(SniProviders.NP_PROV); + return false; + } + + ServerName = NormalizeHostName(ServerName); + InferLocalServerName(); return true; } @@ -700,10 +725,22 @@ private bool InferNamedPipesInformation() InstanceName = PipeToken + PipeName; } - ServerName = IsLocalHost(host) ? Environment.MachineName : host; + // An IPv6 literal must be transcribed before it can appear in a UNC pipe path. + // See GetUncCompatibleHostName for details. + string uncHost = GetUncCompatibleHostName(host); + if (uncHost is null) + { + SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniProxy), EventType.ERR, "Invalid host name '{0}' for Named Pipes.", host); + ReportSNIError(SniProviders.NP_PROV); + return false; + } + + // ServerName drops any brackets because it feeds DNS resolution and SPN + // construction, neither of which accepts the bracketed spelling. + ServerName = IsLocalHost(host) ? Environment.MachineName : NormalizeHostName(host); // Pipe hostname is the hostname after leading \\ which should be passed down as is to open Named Pipe. // For Named Pipes the ServerName makes sense for SPN creation only. - PipeHostName = host; + PipeHostName = uncHost; } catch (UriFormatException) { @@ -729,6 +766,99 @@ private bool InferNamedPipesInformation() private static bool IsLocalHost(string serverName) => ".".Equals(serverName) || "(local)".Equals(serverName) || "localhost".Equals(serverName); + + /// + /// Attempts to interpret a host name as an IPv6 literal, accepting the bracketed form + /// ([::1]) that users may carry over from URL syntax. + /// + private static bool TryParseIPv6Literal(string hostName, out IPAddress address) + { + address = null; + + // A colon is the only character that can make a host name an IPv6 literal, so anything + // without one (host names, IPv4 literals, already-transcribed names) is not a candidate. + if (string.IsNullOrEmpty(hostName) || hostName.IndexOf(':') == -1) + { + return false; + } + + ReadOnlySpan literal = hostName.AsSpan(); + if (literal.Length > 2 && literal[0] == '[' && literal[literal.Length - 1] == ']') + { + literal = literal.Slice(1, literal.Length - 2); + } + + return IPAddress.TryParse(literal, out address) && + address.AddressFamily == AddressFamily.InterNetworkV6; + } + + /// + /// Returns the canonical form of a host name: a bracketed IPv6 literal is unwrapped to its + /// unbracketed form, and every other host name is returned unchanged. + /// + /// + /// feeds DNS resolution and SPN construction, neither of which + /// accepts the bracketed spelling, so the brackets must be dropped before it is used there. + /// + internal static string NormalizeHostName(string hostName) => + TryParseIPv6Literal(hostName, out IPAddress address) ? address.ToString() : hostName; + + /// + /// Converts a host name into a form that can legally appear as the host component of a UNC + /// pipe path (\\host\pipe\sql\query), returning if no such + /// form exists. + /// + /// + /// A UNC host component may never contain a colon, so an IPv6 literal such as ::1 + /// cannot be used directly. Passing one through anyway composes a malformed path like + /// \\::1\pipe\sql\query, which sends the SMB redirector into an SMB session setup + /// whose SPNEGO/NegoEx target name embeds the IPv6 literal; that can fault LSASS on Windows + /// and force a reboot. See https://github.com/dotnet/SqlClient/issues/4523. + /// + /// Windows defines a transcription for exactly this case: replace each : with + /// - and each % (zone index) with s, then append + /// .ipv6-literal.net. For example 2001:db8::1 becomes + /// 2001-db8--1.ipv6-literal.net. See + /// https://learn.microsoft.com/openspecs/windows_protocols/ms-dtyp/62e862f4-2a51-452e-8eeb-dc4ff5ee33cc. + /// + /// Host names without a colon (including IPv4 literals and already-transcribed + /// .ipv6-literal.net names) are returned unchanged. A colon-bearing host name that is + /// not a parseable IPv6 literal has no UNC form and is rejected. + /// + internal static string GetUncCompatibleHostName(string hostName) + { + if (string.IsNullOrEmpty(hostName)) + { + return null; + } + + if (hostName.IndexOf(':') == -1) + { + return hostName; + } + + if (!TryParseIPv6Literal(hostName, out IPAddress address)) + { + return null; + } + + string literal = address.ToString(); + return string.Create(literal.Length + IPv6LiteralHostSuffix.Length, literal, + static (destination, value) => + { + for (int i = 0; i < value.Length; i++) + { + destination[i] = value[i] switch + { + ':' => '-', + '%' => 's', + _ => value[i] + }; + } + + IPv6LiteralHostSuffix.AsSpan().CopyTo(destination.Slice(value.Length)); + }); + } } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/ManagedSni/DataSourceNamedPipesTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/ManagedSni/DataSourceNamedPipesTests.cs new file mode 100644 index 0000000000..b06fe7e56d --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/ManagedSni/DataSourceNamedPipesTests.cs @@ -0,0 +1,212 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#if NET + +using Microsoft.Data.SqlClient.ManagedSni; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests.ManagedSni +{ + /// + /// Regression tests for Named Pipes data source parsing in . + /// + /// A UNC path host component may never contain a colon, so an IPv6 literal server name cannot + /// be used directly. Passing one through anyway composes a malformed pipe path such as + /// \\::1\pipe\sql\query, which sends the SMB redirector into an SMB session setup that + /// can fault LSASS on Windows and force a reboot. Windows instead defines a transcription for + /// this case (2001:db8::1 becomes 2001-db8--1.ipv6-literal.net), which the parser + /// now applies so IPv6 Named Pipes connections keep working. + /// + /// See: https://github.com/dotnet/SqlClient/issues/4523 + /// and https://learn.microsoft.com/openspecs/windows_protocols/ms-dtyp/62e862f4-2a51-452e-8eeb-dc4ff5ee33cc + /// + public class DataSourceNamedPipesTests + { + /// + /// Verifies that an IPv6 literal host is transcribed to its .ipv6-literal.net UNC + /// form, covering both the np:host form and the \\host\pipe\... UNC form, + /// with and without brackets and with a zone index. + /// + [Theory] + [InlineData(@"np:::1", "--1.ipv6-literal.net")] + [InlineData(@"np:[::1]", "--1.ipv6-literal.net")] + [InlineData(@"np:2001:db8::1", "2001-db8--1.ipv6-literal.net")] + [InlineData(@"np:fe80::1%3", "fe80--1s3.ipv6-literal.net")] + [InlineData(@"\\::1\pipe\sql\query", "--1.ipv6-literal.net")] + [InlineData(@"np:\\::1\pipe\sql\query", "--1.ipv6-literal.net")] + [InlineData(@"np:\\[2001:db8::1]\pipe\MSSQL$MYINSTANCE\sql\query", "2001-db8--1.ipv6-literal.net")] + public void ParseServerName_NamedPipesWithIPv6Literal_IsTranscribedToUncForm( + string dataSource, string expectedPipeHostName) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.Equal(DataSource.Protocol.NP, details.ResolvedProtocol); + Assert.Equal(expectedPipeHostName, details.PipeHostName); + // The pipe host name is what reaches the OS, so it must never carry a colon. + Assert.DoesNotContain(":", details.PipeHostName); + } + + /// + /// Verifies that a colon-bearing host that is not a parseable IPv6 literal has no UNC form + /// and is therefore rejected, rather than composing a malformed pipe path. + /// + [Theory] + [InlineData(@"np:not:a:host")] + [InlineData(@"np:2001:db8:::::1")] + [InlineData(@"\\not:a:host\pipe\sql\query")] + public void ParseServerName_NamedPipesWithUnparseableColonHost_IsRejected(string dataSource) + { + Assert.Null(DataSource.ParseServerName(dataSource)); + } + + /// + /// Verifies that IPv6 transcription does not regress legitimate Named Pipes data sources: + /// IPv4 literals, localhost, ., named instances, and explicit UNC pipe paths + /// must still parse and yield an unchanged pipe host name. + /// + [Theory] + [InlineData(@"np:127.0.0.1", "127.0.0.1")] + [InlineData(@"np:localhost", "localhost")] + [InlineData(@"np:.", ".")] + [InlineData(@"np:server\instance", "server")] + [InlineData(@"\\127.0.0.1\pipe\sql\query", "127.0.0.1")] + [InlineData(@"\\.\pipe\MSSQL$MYINSTANCE\sql\query", ".")] + [InlineData(@"\\my-server\pipe\sql\query", "my-server")] + public void ParseServerName_NamedPipesWithValidHost_IsAccepted( + string dataSource, string expectedPipeHostName) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.Equal(DataSource.Protocol.NP, details.ResolvedProtocol); + Assert.Equal(expectedPipeHostName, details.PipeHostName); + Assert.False(string.IsNullOrEmpty(details.PipeName)); + } + + /// + /// Verifies that a Named Pipes data source given without a UNC path still composes the + /// default pipe name, including the MSSQL$<instance> prefix for named instances. + /// These forms are asserted separately from the UNC forms because the UNC path builds its + /// pipe name with , which is platform dependent. + /// + [Theory] + [InlineData(@"np:127.0.0.1", @"sql\query")] + [InlineData(@"np:localhost", @"sql\query")] + [InlineData(@"np:::1", @"sql\query")] + [InlineData(@"np:server\instance", @"MSSQL$instance\sql\query")] + public void ParseServerName_NamedPipesWithoutUncPath_ComposesDefaultPipeName( + string dataSource, string expectedPipeName) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.Equal(expectedPipeName, details.PipeName); + } + + /// + /// Verifies that an IPv6 literal is preserved (unbracketed) in , + /// which feeds DNS resolution and SPN construction, while the pipe host name is transcribed. + /// The bracketed spelling must not survive into because + /// neither DNS nor SPN construction accepts it. + /// + [Theory] + [InlineData(@"np:2001:db8::1")] + [InlineData(@"np:[2001:db8::1]")] + [InlineData(@"np:\\2001:db8::1\pipe\sql\query")] + [InlineData(@"np:\\[2001:db8::1]\pipe\sql\query")] + public void ParseServerName_NamedPipesWithIPv6Literal_PreservesUnbracketedServerNameForSpn(string dataSource) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.Equal("2001:db8::1", details.ServerName); + Assert.Equal("2001-db8--1.ipv6-literal.net", details.PipeHostName); + } + + /// + /// Verifies unwraps bracketed IPv6 literals and + /// leaves every other host name untouched. + /// + [Theory] + [InlineData("[::1]", "::1")] + [InlineData("::1", "::1")] + [InlineData("[2001:db8::1]", "2001:db8::1")] + [InlineData("[fe80::1%3]", "fe80::1%3")] + [InlineData("localhost", "localhost")] + [InlineData("127.0.0.1", "127.0.0.1")] + [InlineData("not:a:host", "not:a:host")] + [InlineData("", "")] + public void NormalizeHostName_ReturnsExpected(string hostName, string expected) + { + Assert.Equal(expected, DataSource.NormalizeHostName(hostName)); + } + + /// + /// Without an explicit protocol prefix, managed SNI defaults to TCP, so an IPv6 literal + /// server name must continue to parse successfully and never reach the Named Pipes path. + /// + [Theory] + [InlineData("::1")] + [InlineData("[::1]")] + [InlineData("fe80::1")] + public void ParseServerName_IPv6LiteralWithoutProtocol_ResolvesToNonNamedPipes(string dataSource) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.NotEqual(DataSource.Protocol.NP, details.ResolvedProtocol); + Assert.Equal(dataSource, details.ServerName); + } + + /// + /// Verifies directly: colon-free host names + /// pass through untouched, IPv6 literals are transcribed per MS-DTYP, and colon-bearing host + /// names with no IPv6 interpretation return . + /// + [Theory] + [InlineData(".", ".")] + [InlineData("localhost", "localhost")] + [InlineData("127.0.0.1", "127.0.0.1")] + [InlineData("my-server.contoso.com", "my-server.contoso.com")] + [InlineData("--1.ipv6-literal.net", "--1.ipv6-literal.net")] + [InlineData("::1", "--1.ipv6-literal.net")] + [InlineData("[::1]", "--1.ipv6-literal.net")] + [InlineData("2001:db8::1", "2001-db8--1.ipv6-literal.net")] + [InlineData("::ffff:1.2.3.4", "--ffff-1.2.3.4.ipv6-literal.net")] + [InlineData("fe80::1%3", "fe80--1s3.ipv6-literal.net")] + public void GetUncCompatibleHostName_ReturnsExpected(string hostName, string expected) + { + Assert.Equal(expected, DataSource.GetUncCompatibleHostName(hostName)); + } + + /// + /// Verifies returns + /// for host names that contain a colon but have no IPv6 interpretation, and for empty input. + /// + [Theory] + [InlineData("not:a:host")] + [InlineData("2001:db8:::::1")] + [InlineData("[:]")] + [InlineData("")] + public void GetUncCompatibleHostName_UnconvertibleHost_ReturnsNull(string hostName) + { + Assert.Null(DataSource.GetUncCompatibleHostName(hostName)); + } + + /// + /// Verifies returns + /// for a null host name. Covered separately from the theory above because xUnit disallows + /// null theory data for a non-nullable string parameter. + /// + [Fact] + public void GetUncCompatibleHostName_Null_ReturnsNull() + { + Assert.Null(DataSource.GetUncCompatibleHostName(null)); + } + } +} + +#endif From ab1f2e4ef5ba444ad9429f80822cb1fdb32c595c Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:19:22 -0700 Subject: [PATCH 07/51] Pipelines | Move CI and PR pipelines to SQL Server 2025 agent images (#4513) * Move CI and PR pipelines to SQL Server 2025 agent images Retire every SQL Server 2022 agent image in favour of the SQL Server 2025 equivalents: ADO-MMS22-SQL22 -> ADO-MMS25-SQL25 ADO-UB22-SQL22 -> ADO-UB24-SQL25 This covers the CI test configurations, the PR pipeline platform list, the Azure package integration test jobs (including the SQL root path, which becomes SQL25RootPath), the Managed Instance jobs, the stress test jobs, and the Linux enclave configuration. Two notes on the change: - CI test stage names derive from the image keys, so stages such as Win22_Sql22 are now named Win25_Sql25. Any branch policies or required status checks that reference the old stage names will need to be updated. - The Linux SQL configuration previously ran on both ADO-UB20-SQL22 and ADO-UB22-SQL22. We only have a single Ubuntu SQL 2025 image, so it now runs on ADO-UB24-SQL25 alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Clarify why the Linux enclave stage key is left unchanged The Linux enclave image key doubles as the generated ADO stage name and is referenced by branch policies and required status checks, so it is kept as-is. The 'Sql19' suffix also remains accurate because these tests target a remote Enclave-enabled SQL Server 2019; only the agent image moved to Ubuntu 24. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Install sqlcmd on Linux agents before configuring SQL Server The SQL Server 2025 agent images ship the engine but not the command line tools, so the Linux SQL configuration steps failed with: ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations. Add a shared 'Install sqlcmd [Linux]' step that reuses an existing sqlcmd when the image provides one and otherwise installs mssql-tools18, then publishes the resolved path via the SqlCmdBin variable. Both the PR and CI Linux configuration steps now run it, and the CI step no longer hardcodes /opt/mssql-tools/bin/sqlcmd, which does not exist on these images. sqlcmd from mssql-tools18 encrypts by default, so the step also publishes SqlCmdTrustArg ('-C') to trust the local server's self-signed certificate. Without it every connection to localhost would fail certificate validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d --- .../sqlclient-ci-managed-instance-stage.yml | 8 +- .../ci/stress/sqlclient-ci-stress-stage.yml | 4 +- .../steps/configure-sql-server-linux-step.yml | 14 +++- .../steps/configure-sql-server-win-step.yml | 2 +- .../steps/install-sqlcmd-linux-step.yml | 83 +++++++++++++++++++ eng/pipelines/dotnet-sqlclient-ci-core.yml | 52 ++++++------ eng/pipelines/pr/sqlclient-pr-pipeline.yml | 14 ++-- .../steps/configure-sqlserver-linux-step.yml | 36 ++++---- .../configure-sqlserver-windows-step.yml | 2 +- .../stages/build-azure-package-ci-stage.yml | 12 +-- 10 files changed, 162 insertions(+), 65 deletions(-) create mode 100644 eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml index 114fef0061..b6a14c54a7 100644 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml @@ -68,7 +68,7 @@ stages: operatingSystem: Windows runtime: ${{ runtime }} useManagedSNI: false - vmImage: ADO-MMS22-SQL22 + vmImage: ADO-MMS25-SQL25 - ${{ each runtime in parameters.netTestRuntimes }}: - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self @@ -81,7 +81,7 @@ stages: operatingSystem: Windows runtime: ${{ runtime }} useManagedSNI: false - vmImage: ADO-MMS22-SQL22 + vmImage: ADO-MMS25-SQL25 - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self parameters: @@ -93,7 +93,7 @@ stages: operatingSystem: Windows runtime: ${{ runtime }} useManagedSNI: true - vmImage: ADO-MMS22-SQL22 + vmImage: ADO-MMS25-SQL25 # ---------------------------------------------------------------------------------------------- # Linux: one job per .NET runtime (managed SNI is always used on non-Windows). @@ -108,4 +108,4 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} operatingSystem: Linux runtime: ${{ runtime }} - vmImage: ADO-UB22-SQL22 + vmImage: ADO-UB24-SQL25 diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml index 689d1a702b..3cbdb9d31e 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml @@ -89,7 +89,7 @@ stages: template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self parameters: saPassword: $(saPassword) - vmImage: ADO-UB22-SQL22 + vmImage: ADO-UB24-SQL25 # ---------------------------------------------------------------------------------------------- # Build and test on Windows @@ -112,7 +112,7 @@ stages: saPassword: $(saPassword) # The Windows images include a suitable .NET Framework runtime, so we don't have to install # one explicitly. - vmImage: ADO-MMS22-SQL22 + vmImage: ADO-MMS25-SQL25 # ---------------------------------------------------------------------------------------------- # Build and test on macOS. diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml index 15de459d50..a397108701 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml @@ -16,8 +16,16 @@ parameters: steps: + # Make sure sqlcmd is available; the SQL Server images do not always ship it. + - template: /eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml@self + # Configure SQL Server. - bash: | + if [ -z "${SQLCMD_BIN:-}" ]; then + echo "ERROR: sqlcmd was not resolved by the 'Install sqlcmd [Linux]' step." + exit 1 + fi + sudo systemctl stop mssql-server # Password for the SA user (required) @@ -39,10 +47,11 @@ steps: do echo Waiting for SQL Server to start... sleep 3s - /opt/mssql-tools/bin/sqlcmd \ + "$SQLCMD_BIN" \ -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -Q "SELECT @@VERSION" 2>/dev/null errstatus=$? ((counter++)) @@ -55,3 +64,6 @@ steps: exit $errstatus fi displayName: 'Configure SQL Server [Linux]' + env: + SQLCMD_BIN: $(SqlCmdBin) + SQLCMD_TRUST_ARG: $(SqlCmdTrustArg) diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml index 5659d45de2..6980c1f33c 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml @@ -5,7 +5,7 @@ ################################################################################# # This step configures an existing SQL Server running on the local Windows host. For example, our -# 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come with SQL Server 2022 pre-installed and +# 1ES Hosted Pool has images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: diff --git a/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml b/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml new file mode 100644 index 0000000000..d5e82f434b --- /dev/null +++ b/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml @@ -0,0 +1,83 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Ensures the SQL Server command line tools are available on a Linux agent. +# +# The SQL Server images in our 1ES pool (for example ADO-UB24-SQL25) ship the SQL Server engine but +# do not always ship the command line tools, so install them when they are missing. +# +# Sets two variables for the remaining steps in the job: +# +# SqlCmdBin - absolute path to the sqlcmd executable. +# SqlCmdTrustArg - '-C' when sqlcmd needs to be told to trust the server's self-signed +# certificate, otherwise empty. + +steps: + + - bash: | + set -u + + if ! command -v sudo >/dev/null 2>&1; then + echo "ERROR: 'sudo' is required to install the SQL Server command line tools." + exit 1 + fi + + find_sqlcmd() { + if command -v sqlcmd >/dev/null 2>&1; then + command -v sqlcmd + return 0 + fi + # mssql-tools default install locations. + local candidate + for candidate in /opt/mssql-tools18/bin/sqlcmd /opt/mssql-tools/bin/sqlcmd; do + if [ -x "$candidate" ]; then + echo "$candidate" + return 0 + fi + done + return 1 + } + + if SQLCMD_BIN="$(find_sqlcmd)"; then + echo "Found existing sqlcmd at '$SQLCMD_BIN'." + else + echo "sqlcmd was not found; installing mssql-tools18..." + + # The Microsoft package repository is usually already configured on these images (that is + # where the engine came from), so try a plain install first and only register the repository + # if that fails. + sudo apt-get update + if ! sudo env ACCEPT_EULA=Y apt-get install -y mssql-tools18 unixodbc-dev; then + # shellcheck disable=SC1091 + . /etc/os-release + echo "Registering the Microsoft package repository for Ubuntu $VERSION_ID..." + curl -sSL -o /tmp/packages-microsoft-prod.deb \ + "https://packages.microsoft.com/config/ubuntu/$VERSION_ID/packages-microsoft-prod.deb" + sudo dpkg -i /tmp/packages-microsoft-prod.deb + sudo apt-get update + sudo env ACCEPT_EULA=Y apt-get install -y mssql-tools18 unixodbc-dev + fi + + if ! SQLCMD_BIN="$(find_sqlcmd)"; then + echo "ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations," + echo " and installing mssql-tools18 did not provide it." + exit 1 + fi + fi + + # sqlcmd from mssql-tools18 (and go-sqlcmd) encrypts by default, so it needs -C to trust the + # local server's self-signed certificate. The older mssql-tools build does not support -C. + case "$SQLCMD_BIN" in + */mssql-tools/bin/sqlcmd) SQLCMD_TRUST_ARG="" ;; + *) SQLCMD_TRUST_ARG="-C" ;; + esac + + echo "Using sqlcmd '$SQLCMD_BIN' with trust argument '$SQLCMD_TRUST_ARG'." + + # Publish for the remaining steps in this job. + echo "##vso[task.setvariable variable=SqlCmdBin]$SQLCMD_BIN" + echo "##vso[task.setvariable variable=SqlCmdTrustArg]$SQLCMD_TRUST_ARG" + displayName: 'Install sqlcmd [Linux]' diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index 3e68e93325..c8c2f72b15 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -299,7 +299,7 @@ stages: testConfigurations: # SQL Server 2016 and 2017 on Windows Server 2022 (x64 only). # x86 testing is intentionally skipped for these legacy SQL versions - # because x86 support is already validated via SQL 2019 and 2022 images. + # because x86 support is already validated via SQL 2019 and 2025 images. ${{ if eq(parameters.runLegacySqlTests, true) }}: # Windows Server 22 with local SQL Server 2016, x64 build platform. windows_sql_16_x64: @@ -407,11 +407,11 @@ stages: SQLRootPath: $(SQL19RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022, x64 build platform. - windows_sql_22_x64: + # Windows Server 25 with local SQL Server 2025, x64 build platform. + windows_sql_25_x64: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22: ADO-MMS22-SQL22 + Win25_Sql25: ADO-MMS25-SQL25 TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -430,14 +430,14 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022, x86 build platform. - windows_sql_22_x86: + # Windows Server 25 with local SQL Server 2025, x86 build platform. + windows_sql_25_x86: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22_x86: ADO-MMS22-SQL22 + Win25_Sql25_x86: ADO-MMS25-SQL25 TargetFrameworks: [net462, net8.0, net9.0] netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -457,14 +457,14 @@ stages: LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) x86TestTargetFrameworks: [net462, net8.0, net9.0] - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022 Named Instance, x64 build platform. - windows_sql_22_named_instance: + # Windows Server 25 with local SQL Server 2025 Named Instance, x64 build platform. + windows_sql_25_named_instance: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22_Named_Instance: ADO-MMS22-SQL22-WITH-NAMED-INSTANCE + Win25_Sql25_Named_Instance: ADO-MMS25-SQL25-WITH-NAMED-INSTANCE TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -476,7 +476,7 @@ stages: TCPConnectionString: $(SQL_TCP_INSTANCE_CONN_STRING) NPConnectionString: $(SQL_NP_INSTANCE_CONN_STRING) SupportsIntegratedSecurity: true - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) instanceName: $(NamedInstance) # Windows Server 2022 and Windows 11, x64 build platform, with Azure SQL Server. @@ -537,12 +537,11 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # Linux Ubuntu 20 and 22 with local SQL Server 2022, x64 build platform. - linux_ub20_22_sql_22: + # Linux Ubuntu 24 with local SQL Server 2025, x64 build platform. + linux_ub24_sql_25: pool: ${{parameters.defaultPoolName }} images: - Ubuntu20_Sql22: ADO-UB20-SQL22 - Ubuntu22_Sql22: ADO-UB22-SQL22 + Ubuntu24_Sql25: ADO-UB24-SQL25 TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] @@ -561,11 +560,11 @@ stages: LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) - # Linux Ubuntu 22 with Azure SQL Server, x64 build platform. + # Linux Ubuntu 24 with Azure SQL Server, x64 build platform. linux_azure_sql: pool: ${{parameters.defaultPoolName }} images: - Ubuntu22_Azure_Sql: ADO-UB22-SQL22 + Ubuntu24_Azure_Sql: ADO-UB24-SQL25 TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] @@ -588,12 +587,12 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # macOS with local SQL Server 2022, x64 build platform. - mac_sql_22: + # macOS with local SQL Server 2025 (docker), x64 build platform. + mac_sql_25: pool: Azure Pipelines hostedPool: true images: - MacOSLatest_Sql22: macos-latest + MacOSLatest_Sql25: macos-latest TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] @@ -642,11 +641,16 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # Linux Ubuntu 22 with remote Enclave-enabled SQL Server 2019, x64 build platform. + # Linux Ubuntu 24 with remote Enclave-enabled SQL Server 2019, x64 build platform. linux_enclave_sql: pool: ADO-CI-AE-1ES-Pool images: - Ubuntu20_Enclave_Sql19: ADO-UB22-Sql22 + # NOTE: This key is also the generated stage name, and is + # referenced by branch policies / required status checks, so it is + # deliberately left unchanged. The 'Sql19' suffix remains + # accurate: these tests target a remote Enclave-enabled SQL Server + # 2019. Only the agent image has moved to Ubuntu 24. + Ubuntu20_Enclave_Sql19: ADO-UB24-SQL25 TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] diff --git a/eng/pipelines/pr/sqlclient-pr-pipeline.yml b/eng/pipelines/pr/sqlclient-pr-pipeline.yml index b72e15ef53..4ab1a31f60 100644 --- a/eng/pipelines/pr/sqlclient-pr-pipeline.yml +++ b/eng/pipelines/pr/sqlclient-pr-pipeline.yml @@ -70,32 +70,32 @@ parameters: default: - displayName: "windows_net462" dotnet: "net462" - image: "ADO-MMS22-SQL22" + image: "ADO-MMS25-SQL25" operatingSystem: "Windows" - displayName: "windows_net8" dotnet: "net8.0" - image: "ADO-MMS22-SQL22" + image: "ADO-MMS25-SQL25" operatingSystem: "Windows" - displayName: "windows_net9" dotnet: "net9.0" - image: "ADO-MMS22-SQL22" + image: "ADO-MMS25-SQL25" operatingSystem: "Windows" - displayName: "windows_net10" dotnet: "net10.0" - image: "ADO-MMS22-SQL22" + image: "ADO-MMS25-SQL25" operatingSystem: "Windows" - displayName: "linux_net8" dotnet: "net8.0" - image: "ADO-UB22-SQL22" + image: "ADO-UB24-SQL25" operatingSystem: "Linux" - displayName: "linux_net9" dotnet: "net9.0" - image: "ADO-UB22-SQL22" + image: "ADO-UB24-SQL25" operatingSystem: "Linux" - displayName: "linux_net10" dotnet: "net10.0" - image: "ADO-UB22-SQL22" + image: "ADO-UB24-SQL25" operatingSystem: "Linux" variables: diff --git a/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml b/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml index 8eb8cc40d6..df625dc35e 100644 --- a/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml +++ b/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml @@ -5,7 +5,7 @@ ################################################################################# # This step configures an existing SQL Server running on the local Linux host. For example, our 1ES -# Hosted Pool has images like ADO-UB20-SQL22 that come with SQL Server 2022 pre-installed and +# Hosted Pool has images like ADO-UB24-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: @@ -16,6 +16,9 @@ parameters: steps: + # Make sure sqlcmd is available; the SQL Server images do not always ship it. + - template: /eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml@self + # Configure SQL Server. - bash: | set -u @@ -30,15 +33,8 @@ steps: exit 1 fi - SQLCMD_BIN="" - if command -v sqlcmd >/dev/null 2>&1; then - SQLCMD_BIN="$(command -v sqlcmd)" - elif [ -x /opt/mssql-tools18/bin/sqlcmd ]; then - SQLCMD_BIN="/opt/mssql-tools18/bin/sqlcmd" - elif [ -x /opt/mssql-tools/bin/sqlcmd ]; then - SQLCMD_BIN="/opt/mssql-tools/bin/sqlcmd" - else - echo "ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations." + if [ -z "${SQLCMD_BIN:-}" ]; then + echo "ERROR: sqlcmd was not resolved by the 'Install sqlcmd [Linux]' step." exit 1 fi @@ -79,6 +75,7 @@ steps: -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -Q "SELECT @@VERSION" 2>/dev/null errstatus=$? ((counter++)) @@ -91,6 +88,9 @@ steps: exit $errstatus fi displayName: 'Configure SQL Server [Linux]' + env: + SQLCMD_BIN: $(SqlCmdBin) + SQLCMD_TRUST_ARG: $(SqlCmdTrustArg) - bash: | set -u @@ -98,15 +98,8 @@ steps: SCRIPT_PATH="$(Build.SourcesDirectory)/tools/testsql/createNorthwindDb.sql" MSSQL_SA_PW="${{ parameters.saPassword }}" - SQLCMD_BIN="" - if command -v sqlcmd >/dev/null 2>&1; then - SQLCMD_BIN="$(command -v sqlcmd)" - elif [ -x /opt/mssql-tools18/bin/sqlcmd ]; then - SQLCMD_BIN="/opt/mssql-tools18/bin/sqlcmd" - elif [ -x /opt/mssql-tools/bin/sqlcmd ]; then - SQLCMD_BIN="/opt/mssql-tools/bin/sqlcmd" - else - echo "ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations." + if [ -z "${SQLCMD_BIN:-}" ]; then + echo "ERROR: sqlcmd was not resolved by the 'Install sqlcmd [Linux]' step." exit 1 fi @@ -114,6 +107,7 @@ steps: -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -Q "IF DB_ID(N'Northwind') IS NOT NULL BEGIN ALTER DATABASE [Northwind] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [Northwind]; END" \ -b @@ -121,7 +115,11 @@ steps: -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -i "$SCRIPT_PATH" \ -b displayName: 'Create Northwind Database [Linux]' retryCountOnTaskFailure: 1 + env: + SQLCMD_BIN: $(SqlCmdBin) + SQLCMD_TRUST_ARG: $(SqlCmdTrustArg) diff --git a/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml b/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml index 8e07120b4b..405582d655 100644 --- a/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml +++ b/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml @@ -5,7 +5,7 @@ ################################################################################# # This step configures an existing SQL Server running on the local Windows host. For example, our -# 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come with SQL Server 2022 pre-installed and +# 1ES Hosted Pool has images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: diff --git a/eng/pipelines/stages/build-azure-package-ci-stage.yml b/eng/pipelines/stages/build-azure-package-ci-stage.yml index 0713030b76..b907d2fcda 100644 --- a/eng/pipelines/stages/build-azure-package-ci-stage.yml +++ b/eng/pipelines/stages/build-azure-package-ci-stage.yml @@ -47,8 +47,8 @@ parameters: # # Any pool specified here must contain images with the following names: # - # - ADO-MMS22-SQL22 - # - ADO-UB22-SQL22 + # - ADO-MMS25-SQL25 + # - ADO-UB24-SQL25 # default: $(ci_var_defaultPoolName) @@ -187,7 +187,7 @@ stages: - template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self parameters: saPassword: $(saPassword) - vmImage: ADO-UB22-SQL22 + vmImage: ADO-UB24-SQL25 # ------------------------------------------------------------------------ # Build and test on Windows @@ -241,11 +241,11 @@ stages: # These variables are from an Azure DevOps Library variable # group. fileStreamDirectory: $(FileStreamDirectory) - sqlRootPath: $(SQL22RootPath) - # The ADO-MMS22-SQL22 image includes a local SQL Server that supports + SQLRootPath: $(SQL25RootPath) + # The ADO-MMS25-SQL25 image includes a local SQL Server that supports # integrated security. supportsIntegratedSecurity: true - vmImage: ADO-MMS22-SQL22 + vmImage: ADO-MMS25-SQL25 # ------------------------------------------------------------------------ # Build and test on macOS. From ef2ca70e25c086e3b095b562e3ba983482e0376e Mon Sep 17 00:00:00 2001 From: priyankatiwari08 Date: Wed, 2 Sep 2026 13:45:40 +0530 Subject: [PATCH 08/51] Stop the localization pipeline from opening duplicate PRs (#4615) * Add idempotent localization PR script to stop duplicate PRs The scheduled Localization-CI pipeline opens a brand-new GitHub PR on every run because its inline "Open PR on GitHub" step pushes a timestamped branch (dev/automation/onelocbuild-) and never checks whether an equivalent PR is already open. Four byte-for-byte identical PRs (#4607, #4612, #4613, #4614) accumulated as a result. Add eng/pipelines/scripts/Open-LocalizationPr.ps1 as a reusable, idempotent replacement for that inline step: - Uses a stable branch name, rebuilt from the base branch each run, so no timestamped branch proliferation and no commit accumulation. - Exits without pushing or calling the GitHub API when the localized resources are identical to the base branch. - Skips the force-push when the remote branch already holds the exact same tree on top of the same base. - Reuses an already-open pull request (PATCH) instead of opening a second one, and otherwise opens exactly one new PR. Includes Pester v5 tests covering the de-duplication contract with git and Invoke-RestMethod mocked. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cb6c5b07-fb77-43bc-a4aa-a9d39aa04438 * Add -DryRun switch to Open-LocalizationPr.ps1 Allows validating the pipeline wiring (paths, token scopes, OneLocBuild output, existing-PR lookup) from a feature branch without pushing a branch or creating/updating a pull request in the public repository. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cb6c5b07-fb77-43bc-a4aa-a9d39aa04438 * Address review feedback on Open-LocalizationPr - Authenticate git via GIT_CONFIG_* environment config instead of embedding the token in the remote URL, so it never reaches .git/config, a process command line, or Invoke-Git exception messages. Cleared in the finally block. - Read GitHub error bodies from $_.ErrorDetails first, since PowerShell 7 exposes an HttpResponseMessage with no GetResponseStream(); keep the stream path as a Windows PowerShell fallback so 4xx bodies are no longer dropped. - Replace the unconditional force-push with --force-with-lease pinned to the remote SHA observed earlier in the run, so an overlapping run fails instead of discarding a concurrent localization result. Adds 4 tests (17 total, all passing). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cb6c5b07-fb77-43bc-a4aa-a9d39aa04438 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cb6c5b07-fb77-43bc-a4aa-a9d39aa04438 --- eng/pipelines/scripts/Open-LocalizationPr.ps1 | 454 ++++++++++++++++++ .../tests/Open-LocalizationPr.Tests.ps1 | 357 ++++++++++++++ eng/pipelines/scripts/tests/README.md | 29 ++ 3 files changed, 840 insertions(+) create mode 100644 eng/pipelines/scripts/Open-LocalizationPr.ps1 create mode 100644 eng/pipelines/scripts/tests/Open-LocalizationPr.Tests.ps1 create mode 100644 eng/pipelines/scripts/tests/README.md diff --git a/eng/pipelines/scripts/Open-LocalizationPr.ps1 b/eng/pipelines/scripts/Open-LocalizationPr.ps1 new file mode 100644 index 0000000000..0e7411ec7e --- /dev/null +++ b/eng/pipelines/scripts/Open-LocalizationPr.ps1 @@ -0,0 +1,454 @@ +<# +.SYNOPSIS + Publishes OneLocBuild-generated localized resource files to GitHub as a + single, continuously-updated pull request. + +.DESCRIPTION + The Localization-CI pipeline regenerates the localized `Strings.*.resx` + files on every scheduled run. Because the generated content is identical + until a previous localization PR is merged, naively opening a new PR per + run produces a pile of byte-for-byte duplicate PRs. + + This script makes the publish step idempotent: + + 1. Clones the target GitHub repository at the base branch. + 2. Copies the freshly generated localized resource files into the clone. + 3. If the content matches the base branch, exits without doing anything. + 4. Otherwise commits onto a *stable* branch name, force-pushes it, and + reuses the existing open pull request if one is already present. + + The net effect is at most one open localization PR at any time. Subsequent + runs refresh that PR in place instead of opening another one. + +.PARAMETER GitHubRepository + The target repository in "owner/repo" form. A trailing ".git" is tolerated + so the existing $(GitHubRepository) pipeline variable can be passed as-is. + +.PARAMETER AccessToken + A GitHub token with "contents: write" and "pull requests: write" on the + target repository. Defaults to the GITHUB_TOKEN environment variable. + +.PARAMETER SourceDirectory + The directory holding the OneLocBuild output. Localized files are resolved + relative to this path using ResourcesPath. Defaults to the current + directory. + +.PARAMETER WorkingDirectory + The directory the target repository is cloned into. Defaults to a new + "loc-pr-" folder under the system temp path, which is removed when + the script finishes. + +.PARAMETER BaseBranch + The branch the pull request targets. Defaults to "main". + +.PARAMETER BranchName + The stable branch the localized files are published to. Reusing one branch + name across runs is what allows the pull request to be reused. Defaults to + "dev/automation/onelocbuild". + +.PARAMETER ResourcesPath + Repository-relative path to the resources folder. Defaults to the + Microsoft.Data.SqlClient resources folder. + +.PARAMETER ResourceFilePattern + Filename pattern for the localized resource files to publish. Defaults to + "Strings.*.resx", which matches the localized files but not the English + "Strings.resx" source. + +.PARAMETER DryRun + Performs every read-only step - clone, copy, change detection and the + existing-pull-request lookup - but does not push the branch and does not + create or update a pull request. Use this to validate pipeline wiring + (paths, token scopes, OneLocBuild output) without publishing anything. + +.NOTES + Intended to be invoked from the internal Localization-CI pipeline. It is + safe to re-run: with no localization changes pending it is a no-op, and + with changes pending it converges on a single open pull request. +#> + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# See the LICENSE file in the project root for more information. + +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [string]$GitHubRepository, + + [string]$AccessToken = $env:GITHUB_TOKEN, + + [string]$SourceDirectory = (Get-Location).Path, + + [string]$WorkingDirectory, + + [string]$BaseBranch = 'main', + + [string]$BranchName = 'dev/automation/onelocbuild', + + [string]$ResourcesPath = 'src/Microsoft.Data.SqlClient/src/Resources', + + [string]$ResourceFilePattern = 'Strings.*.resx', + + [switch]$DryRun +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Several git invocations below use the exit code as a signal (for example +# "git diff --cached --quiet" returns 1 when there are staged changes). On +# PowerShell 7.3+ that would otherwise be turned into a terminating error. +$PSNativeCommandUseErrorActionPreference = $false + +$PrTitle = '[Scheduled Run] Localized resource files from OneLocBuild' + +#region Helper Functions + +function Get-RepositorySlug { + <# + .SYNOPSIS + Normalizes a repository reference into "owner/repo" form. + #> + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$Repository + ) + + $slug = $Repository.Trim().TrimEnd('/') + if ($slug.EndsWith('.git', [StringComparison]::OrdinalIgnoreCase)) { + $slug = $slug.Substring(0, $slug.Length - 4) + } + + if ($slug -notmatch '^[^/\s]+/[^/\s]+$') { + throw "GitHubRepository must be in 'owner/repo' form, but was '$Repository'." + } + + return $slug +} + +function Get-PullRequestBody { + <# + .SYNOPSIS + Builds the pull request description. + #> + param( + [Parameter(Mandatory)][string]$Branch, + [Parameter(Mandatory)][datetime]$UpdatedUtc + ) + + $timestamp = $UpdatedUtc.ToString('yyyy-MM-dd HH:mm') + + return @" +Automated PR created from the OneLocBuild scheduled pipeline run. + +Contains updated localized ``Strings.*.resx`` resource files. + +This pull request is refreshed in place by every scheduled localization run, so +there is only ever one open localization PR. Merging it lets the next run start +from a clean base; leaving it open simply keeps it up to date. + +- Branch: ``$Branch`` +- Last updated: $timestamp UTC +"@ +} + +function Invoke-GitHubApi { + <# + .SYNOPSIS + Calls a GitHub REST API endpoint, surfacing the response body on error. + #> + param( + [Parameter(Mandatory)][string]$Uri, + [string]$Method = 'GET', + [object]$Body = $null, + [Parameter(Mandatory)][hashtable]$Headers + ) + + $params = @{ + Uri = $Uri + Method = $Method + Headers = $Headers + } + + if ($null -ne $Body) { + $params['Body'] = ($Body | ConvertTo-Json -Depth 10 -Compress) + $params['ContentType'] = 'application/json' + } + + try { + return Invoke-RestMethod @params -ErrorAction Stop + } + catch { + $status = $null + $responseBody = '' + + # PowerShell 7 already carries the response body on the error record and + # exposes an HttpResponseMessage, which has no GetResponseStream(). Windows + # PowerShell leaves ErrorDetails empty but does expose the stream, so try + # the error record first and keep the stream as a fallback. + if ($null -ne $_.ErrorDetails -and -not [string]::IsNullOrWhiteSpace($_.ErrorDetails.Message)) { + $responseBody = $_.ErrorDetails.Message + } + + if ($_.Exception.PSObject.Properties.Name -contains 'Response' -and $null -ne $_.Exception.Response) { + $status = $_.Exception.Response.StatusCode.value__ + + if ([string]::IsNullOrWhiteSpace($responseBody)) { + try { + $stream = $_.Exception.Response.GetResponseStream() + if ($null -ne $stream) { + $reader = New-Object System.IO.StreamReader($stream) + try { $responseBody = $reader.ReadToEnd() } finally { $reader.Dispose() } + } + } + catch { + # No readable stream (PowerShell 7) or it was already consumed; + # the status code alone is still useful. + } + } + } + + Write-Host "##vso[task.logissue type=error]GitHub API $Method $Uri failed (HTTP $status)." + if ($responseBody) { + Write-Host "##vso[task.logissue type=error]Response: $responseBody" + } + + throw + } +} + +function Invoke-Git { + <# + .SYNOPSIS + Runs a git command and throws if it reports failure. + #> + param( + [Parameter(Mandatory, ValueFromRemainingArguments)][string[]]$Arguments + ) + + & git @Arguments + if ($LASTEXITCODE -ne 0) { + throw "git $($Arguments -join ' ') failed with exit code $LASTEXITCODE." + } +} + +#endregion + +#region Validation + +if ([string]::IsNullOrWhiteSpace($AccessToken)) { + throw 'A GitHub access token is required. Set GITHUB_TOKEN or pass -AccessToken.' +} + +$repoSlug = Get-RepositorySlug -Repository $GitHubRepository +$repoOwner = $repoSlug.Split('/')[0] + +$sourceResources = Join-Path $SourceDirectory $ResourcesPath +if (-not (Test-Path -LiteralPath $sourceResources)) { + throw "Localized resources folder not found at '$sourceResources'." +} + +$localizedFiles = @(Get-ChildItem -Path $sourceResources -Filter $ResourceFilePattern -File) +if ($localizedFiles.Count -eq 0) { + throw "No files matching '$ResourceFilePattern' were found in '$sourceResources'. Did the OneLocBuild step run?" +} + +$ownedWorkingDirectory = $false +if ([string]::IsNullOrWhiteSpace($WorkingDirectory)) { + $WorkingDirectory = Join-Path ([System.IO.Path]::GetTempPath()) "loc-pr-$([guid]::NewGuid().ToString('n'))" + $ownedWorkingDirectory = $true +} + +$headers = @{ + 'Authorization' = "Bearer $AccessToken" + 'Accept' = 'application/vnd.github+json' + 'X-GitHub-Api-Version' = '2022-11-28' + 'User-Agent' = 'SqlClient-DevOps' +} + +#endregion + +Write-Host '=== Publish localized resources to GitHub ===' +Write-Host "Repository : $repoSlug" +Write-Host "Base : $BaseBranch" +Write-Host "Branch : $BranchName" +Write-Host "Resources : $($localizedFiles.Count) file(s) from $sourceResources" +if ($DryRun) { + Write-Host 'Mode : DRY RUN (no push, no pull request changes)' +} +Write-Host '' + +$originalLocation = Get-Location + +try { + #region Clone and stage + + if (Test-Path -LiteralPath $WorkingDirectory) { + Remove-Item -LiteralPath $WorkingDirectory -Recurse -Force + } + + # Authenticate through git's environment-based config rather than the remote + # URL, so the token never lands in .git/config, on a process command line, or + # in Invoke-Git's exception messages. GIT_CONFIG_* is honoured by git 2.31+ + # and applies to clone, fetch and push alike. + $basicAuth = [Convert]::ToBase64String( + [Text.Encoding]::ASCII.GetBytes("x-access-token:$AccessToken")) + $env:GIT_CONFIG_COUNT = '1' + $env:GIT_CONFIG_KEY_0 = 'http.https://github.com/.extraheader' + $env:GIT_CONFIG_VALUE_0 = "AUTHORIZATION: basic $basicAuth" + $cloneUrl = "https://github.com/$repoSlug.git" + + Write-Host "Cloning $repoSlug@$BaseBranch..." + Invoke-Git clone --branch $BaseBranch --quiet $cloneUrl $WorkingDirectory + + Set-Location -LiteralPath $WorkingDirectory + + Invoke-Git config user.email 'sqlclient@microsoft.com' + Invoke-Git config user.name 'SqlClient DevOps' + + $baseSha = (& git rev-parse HEAD).Trim() + Write-Host "Base HEAD : $baseSha" + + # Fetch the automation branch if it already exists so we can tell an actual + # content change apart from a re-run that would produce an identical commit. + & git fetch origin "refs/heads/${BranchName}:refs/remotes/origin/$BranchName" --quiet 2>&1 | Out-Null + $remoteBranchSha = (& git rev-parse --verify --quiet "refs/remotes/origin/$BranchName") + $remoteBranchExists = -not [string]::IsNullOrWhiteSpace($remoteBranchSha) + + if ($remoteBranchExists) { + Write-Host "Branch HEAD: $($remoteBranchSha.Trim())" + } + + $targetResources = Join-Path $WorkingDirectory $ResourcesPath + if (-not (Test-Path -LiteralPath $targetResources)) { + throw "Resources folder '$ResourcesPath' does not exist in $repoSlug@$BaseBranch." + } + + Copy-Item -Path (Join-Path $sourceResources $ResourceFilePattern) -Destination $targetResources -Force + + Invoke-Git add --all -- $ResourcesPath + + & git diff --cached --quiet + if ($LASTEXITCODE -eq 0) { + Write-Host '' + Write-Host "No localization changes relative to '$BaseBranch'. Nothing to publish." + exit 0 + } + + #endregion + + #region Commit and push + + Write-Host '' + Write-Host "Committing localized resources onto '$BranchName'..." + + # Always rebuild the branch from the current base so the pull request stays + # mergeable, rather than accumulating commits run over run. + Invoke-Git checkout -B $BranchName --quiet + Invoke-Git commit --quiet --message $PrTitle + + $newTree = (& git rev-parse 'HEAD^{tree}').Trim() + + $branchUpToDate = $false + if ($remoteBranchExists) { + $remoteTree = (& git rev-parse "refs/remotes/origin/$BranchName^{tree}" 2>$null) + $remoteParent = (& git rev-parse --verify --quiet "refs/remotes/origin/$BranchName^") + + $branchUpToDate = + $null -ne $remoteTree -and + $remoteTree.Trim() -eq $newTree -and + $null -ne $remoteParent -and + $remoteParent.Trim() -eq $baseSha + } + + if ($branchUpToDate) { + Write-Host "Branch '$BranchName' already has these exact changes on top of '$BaseBranch'. Skipping push." + } + elseif ($DryRun) { + Write-Host "[DryRun] Would push '$BranchName' to $repoSlug." + } + else { + # Pin the push to the remote SHA observed above so an overlapping run that + # already updated the branch makes this push fail loudly, instead of + # silently discarding the other run's localization result. + $pushArgs = @('push', 'origin', "${BranchName}:refs/heads/$BranchName", '--quiet') + if ($remoteBranchExists) { + $pushArgs += "--force-with-lease=refs/heads/${BranchName}:$($remoteBranchSha.Trim())" + } + + Invoke-Git @pushArgs + Write-Host "Pushed '$BranchName'." + } + + #endregion + + #region Create or reuse the pull request + + Write-Host '' + Write-Host 'Checking for an existing open pull request...' + + $encodedHead = [Uri]::EscapeDataString("${repoOwner}:$BranchName") + $encodedBase = [Uri]::EscapeDataString($BaseBranch) + $listUri = "https://api.github.com/repos/$repoSlug/pulls?state=open&head=$encodedHead&base=$encodedBase" + + $openPrs = @(Invoke-GitHubApi -Uri $listUri -Headers $headers) + $existingPr = $openPrs | Select-Object -First 1 + + $body = Get-PullRequestBody -Branch $BranchName -UpdatedUtc ([datetime]::UtcNow) + + if ($existingPr) { + $prNumber = $existingPr.number + + if ($DryRun) { + Write-Host "[DryRun] Would refresh open PR #$prNumber ($($existingPr.html_url))." + } + else { + Write-Host "Reusing open PR #$prNumber - refreshing its description." + + $patchUri = "https://api.github.com/repos/$repoSlug/pulls/$prNumber" + Invoke-GitHubApi -Uri $patchUri -Method 'PATCH' -Headers $headers -Body @{ + title = $PrTitle + body = $body + } | Out-Null + + Write-Host "Pull request updated: $($existingPr.html_url)" + } + } + elseif ($DryRun) { + Write-Host "[DryRun] No open pull request found. Would create one ($BranchName -> $BaseBranch)." + } + else { + Write-Host 'No open pull request found. Creating one...' + + $createUri = "https://api.github.com/repos/$repoSlug/pulls" + $newPr = Invoke-GitHubApi -Uri $createUri -Method 'POST' -Headers $headers -Body @{ + title = $PrTitle + head = $BranchName + base = $BaseBranch + body = $body + } + + Write-Host "Pull request created: $($newPr.html_url)" + } + + #endregion + + Write-Host '' + if ($DryRun) { + Write-Host '=== Done (dry run - nothing was published) ===' + } + else { + Write-Host '=== Done ===' + } +} +finally { + Set-Location -LiteralPath $originalLocation + + Remove-Item Env:\GIT_CONFIG_COUNT -ErrorAction SilentlyContinue + Remove-Item Env:\GIT_CONFIG_KEY_0 -ErrorAction SilentlyContinue + Remove-Item Env:\GIT_CONFIG_VALUE_0 -ErrorAction SilentlyContinue + + if ($ownedWorkingDirectory -and (Test-Path -LiteralPath $WorkingDirectory)) { + Remove-Item -LiteralPath $WorkingDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/eng/pipelines/scripts/tests/Open-LocalizationPr.Tests.ps1 b/eng/pipelines/scripts/tests/Open-LocalizationPr.Tests.ps1 new file mode 100644 index 0000000000..9f99666c9a --- /dev/null +++ b/eng/pipelines/scripts/tests/Open-LocalizationPr.Tests.ps1 @@ -0,0 +1,357 @@ +<# +.SYNOPSIS + Pester tests for Open-LocalizationPr.ps1. + +.DESCRIPTION + These tests focus on the de-duplication behaviour: a scheduled run must + reuse the existing open localization pull request rather than opening a new + one, and must do nothing at all when there is no localization delta. + + 'git' and 'Invoke-RestMethod' are mocked, so no network or repository + access is required. +#> + +BeforeAll { + $global:scriptPath = Join-Path $PSScriptRoot '..' 'Open-LocalizationPr.ps1' + $global:resourcesPath = 'src/Microsoft.Data.SqlClient/src/Resources' + + function New-TempDirectoryPath { + return Join-Path ([System.IO.Path]::GetTempPath()) "loc-test-$([guid]::NewGuid().ToString('n'))" + } + + function New-SourceTree { + $root = New-TempDirectoryPath + $resources = Join-Path $root $global:resourcesPath + New-Item -ItemType Directory -Path $resources -Force | Out-Null + + foreach ($culture in @('de', 'fr', 'ja')) { + Set-Content -Path (Join-Path $resources "Strings.$culture.resx") -Value '' + } + + return $root + } + + function Get-RestCallCount { + param( + [Parameter(Mandatory)][string]$Method, + [Parameter(Mandatory)][string]$UriPattern + ) + + return @($global:restCalls | Where-Object { $_.Method -eq $Method -and $_.Uri -like $UriPattern }).Count + } + + function Get-RestCallsTo { + param( + [Parameter(Mandatory)][string]$Method, + [Parameter(Mandatory)][string]$UriPattern + ) + + return , @($global:restCalls | Where-Object { $_.Method -eq $Method -and $_.Uri -like $UriPattern }) + } +} + +Describe 'Open-LocalizationPr.ps1' { + + BeforeAll { + Mock -CommandName 'git' -MockWith { + $joined = $args -join ' ' + $global:gitCalls += $joined + $global:LASTEXITCODE = 0 + + if ($joined -like 'clone*') { + # Stand in for a real clone so the script finds the expected layout. + New-Item -ItemType Directory -Force -Path (Join-Path $global:workDir $global:resourcesPath) | Out-Null + return + } + + if ($joined -like 'diff --cached --quiet*') { + $global:LASTEXITCODE = $global:diffExitCode + return + } + + if ($joined -like 'rev-parse HEAD^{tree}*') { return $global:localTree } + if ($joined -like 'rev-parse HEAD*') { return $global:baseSha } + if ($joined -like 'rev-parse *origin/*^{tree}*') { return $global:remoteTree } + if ($joined -like 'rev-parse *origin/*^') { return $global:remoteParent } + if ($joined -like 'rev-parse *origin/*') { return $global:remoteSha } + + return $null + } + + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:restCalls += @{ Method = $Method; Uri = $Uri; Body = $Body } + + if ($Method -eq 'GET') { + return $global:openPullRequests + } + + return @{ number = 999; html_url = 'https://github.com/dotnet/SqlClient/pull/999' } + } + } + + BeforeEach { + $global:gitCalls = @() + $global:restCalls = @() + $global:diffExitCode = 1 # by default, there is a localization delta + $global:baseSha = 'base000000' + $global:localTree = 'tree111111' + $global:remoteSha = $null # by default, the branch does not exist yet + $global:remoteTree = $null + $global:remoteParent = $null + $global:openPullRequests = @() + + $global:sourceDir = New-SourceTree + $global:workDir = New-TempDirectoryPath + } + + AfterEach { + foreach ($path in @($global:sourceDir, $global:workDir)) { + if ($path -and (Test-Path -LiteralPath $path)) { + Remove-Item -LiteralPath $path -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + + Context 'Input validation' { + + It 'Requires an access token' { + { & $global:scriptPath -GitHubRepository 'dotnet/SqlClient' -AccessToken '' -SourceDirectory $global:sourceDir } | + Should -Throw '*access token is required*' + } + + It 'Rejects a repository that is not in owner/repo form' { + { & $global:scriptPath -GitHubRepository 'SqlClient' -AccessToken 'token' -SourceDirectory $global:sourceDir } | + Should -Throw "*'owner/repo' form*" + } + + It 'Fails when the localized resources are missing' { + Remove-Item -Path (Join-Path $global:sourceDir $global:resourcesPath 'Strings.*.resx') -Force + + { & $global:scriptPath -GitHubRepository 'dotnet/SqlClient' -AccessToken 'token' -SourceDirectory $global:sourceDir } | + Should -Throw '*Did the OneLocBuild step run?*' + } + } + + Context 'When there is no localization delta' { + + It 'Does not push a branch or touch the GitHub API' { + $global:diffExitCode = 0 + + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $global:restCalls.Count | Should -Be 0 + @($global:gitCalls | Where-Object { $_ -like 'push*' }).Count | Should -Be 0 + @($global:gitCalls | Where-Object { $_ -like 'commit*' }).Count | Should -Be 0 + } + } + + Context 'When a localization pull request is already open' { + + BeforeEach { + $global:openPullRequests = @( + @{ number = 4612; html_url = 'https://github.com/dotnet/SqlClient/pull/4612' } + ) + } + + It 'Updates the existing pull request instead of opening another one' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + (Get-RestCallCount -Method 'POST' -UriPattern '*/pulls') | Should -Be 0 + (Get-RestCallCount -Method 'PATCH' -UriPattern '*/pulls/4612') | Should -Be 1 + } + + It 'Looks the pull request up by the stable head branch' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $lookups = Get-RestCallsTo -Method 'GET' -UriPattern '*/pulls?*' + $lookups.Count | Should -Be 1 + $lookups[0].Uri | Should -BeLike '*state=open*' + $lookups[0].Uri | Should -BeLike '*dotnet%3Adev%2Fautomation%2Fonelocbuild*' + } + } + + Context 'When no localization pull request is open' { + + It 'Creates exactly one pull request' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + (Get-RestCallCount -Method 'POST' -UriPattern '*/repos/dotnet/SqlClient/pulls') | Should -Be 1 + } + + It 'Pushes a stable branch name rather than a timestamped one' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $push = @($global:gitCalls | Where-Object { $_ -like 'push*' }) + $push.Count | Should -Be 1 + $push[0] | Should -BeLike '*dev/automation/onelocbuild:refs/heads/dev/automation/onelocbuild*' + $push[0] | Should -Not -Match 'onelocbuild-\d' + } + + It 'Normalizes a repository value that carries a .git suffix' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient.git' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + (Get-RestCallCount -Method 'POST' -UriPattern 'https://api.github.com/repos/dotnet/SqlClient/pulls') | + Should -Be 1 + } + } + + Context 'When the branch already carries the identical change' { + + BeforeEach { + $global:remoteSha = 'remote0000' + $global:remoteTree = $global:localTree + $global:remoteParent = $global:baseSha + $global:openPullRequests = @( + @{ number = 4612; html_url = 'https://github.com/dotnet/SqlClient/pull/4612' } + ) + } + + It 'Skips the force-push but still keeps the pull request current' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + @($global:gitCalls | Where-Object { $_ -like 'push*' }).Count | Should -Be 0 + (Get-RestCallCount -Method 'PATCH' -UriPattern '*/pulls/4612') | Should -Be 1 + (Get-RestCallCount -Method 'POST' -UriPattern '*/pulls') | Should -Be 0 + } + } + + Context 'When pushing over an existing remote branch' { + + BeforeEach { + $global:remoteSha = 'remote0000' + $global:remoteTree = 'differenttree' + $global:remoteParent = 'staleparent' + $global:openPullRequests = @( + @{ number = 4612; html_url = 'https://github.com/dotnet/SqlClient/pull/4612' } + ) + } + + It 'Leases the push against the observed remote SHA' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $push = @($global:gitCalls | Where-Object { $_ -like 'push*' }) + $push.Count | Should -Be 1 + $push[0] | Should -BeLike '*--force-with-lease=refs/heads/dev/automation/onelocbuild:remote0000*' + $push[0] | Should -Not -Match '(^|\s)--force(\s|$)' + } + } + + Context 'When the remote branch does not exist yet' { + + It 'Pushes without forcing' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $push = @($global:gitCalls | Where-Object { $_ -like 'push*' }) + $push.Count | Should -Be 1 + $push[0] | Should -Not -Match '--force' + } + } + + Context 'Credential handling' { + + It 'Keeps the access token out of the git remote URL' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'super-secret-token-value' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $global:gitCalls | Should -Not -Match 'super-secret-token-value' + @($global:gitCalls | Where-Object { $_ -like 'clone*' })[0] | + Should -BeLike '*https://github.com/dotnet/SqlClient.git*' + } + + It 'Clears the git credential environment afterwards' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $env:GIT_CONFIG_COUNT | Should -BeNullOrEmpty + $env:GIT_CONFIG_VALUE_0 | Should -BeNullOrEmpty + } + } + + Context 'When running in dry-run mode' { + + It 'Neither pushes nor creates a pull request' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir ` + -DryRun + + @($global:gitCalls | Where-Object { $_ -like 'push*' }).Count | Should -Be 0 + (Get-RestCallCount -Method 'POST' -UriPattern '*/pulls') | Should -Be 0 + (Get-RestCallCount -Method 'PATCH' -UriPattern '*/pulls/*') | Should -Be 0 + } + + It 'Still performs the existing pull request lookup' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir ` + -DryRun + + (Get-RestCallCount -Method 'GET' -UriPattern '*/pulls?*') | Should -Be 1 + } + + It 'Does not update an existing pull request' { + $global:remoteSha = 'remote0000' + $global:remoteTree = 'differenttree' + $global:remoteParent = 'oldbase000' + $global:openPullRequests = @( + @{ number = 4612; html_url = 'https://github.com/dotnet/SqlClient/pull/4612' } + ) + + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir ` + -DryRun + + @($global:gitCalls | Where-Object { $_ -like 'push*' }).Count | Should -Be 0 + (Get-RestCallCount -Method 'PATCH' -UriPattern '*/pulls/4612') | Should -Be 0 + } + } +} diff --git a/eng/pipelines/scripts/tests/README.md b/eng/pipelines/scripts/tests/README.md new file mode 100644 index 0000000000..8906214afd --- /dev/null +++ b/eng/pipelines/scripts/tests/README.md @@ -0,0 +1,29 @@ +# Pipeline Script Tests + +Pester tests for the PowerShell helpers under `eng/pipelines/scripts/`. + +## Prerequisites + +These tests require **Pester v5 or later**: + +```powershell +Install-Module Pester -MinimumVersion 5.0 -Scope CurrentUser -Force -SkipPublisherCheck +``` + +## Running the tests + +```powershell +Import-Module Pester -MinimumVersion 5.0 +Invoke-Pester ./eng/pipelines/scripts/tests/ +``` + +Add `-Output Detailed` to see per-test results. + +## Test files + +| File | Covers | +| ---- | ------ | +| `Open-LocalizationPr.Tests.ps1` | `Open-LocalizationPr.ps1` — de-duplication of the scheduled localization pull request. | + +`git` and `Invoke-RestMethod` are mocked, so the tests never touch the network +or a real repository. From 83656896d793e3130541f3e6dc7b2f0464fd4ac8 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Wed, 2 Sep 2026 04:47:05 -0700 Subject: [PATCH 09/51] Add .NET 10 test coverage to the CI-SqlClient pipeline (#4514) Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d --- eng/pipelines/dotnet-sqlclient-ci-core.yml | 28 ++++++++++++--- ...qlclient-ci-package-reference-pipeline.yml | 5 +++ ...qlclient-ci-project-reference-pipeline.yml | 35 ++++++++++++++++--- .../sqlclient-pr-package-ref-pipeline.yml | 5 +++ .../sqlclient-pr-project-ref-pipeline.yml | 5 +++ 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index c8c2f72b15..335e98b575 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -25,6 +25,26 @@ parameters: type: object default: [net8.0, net9.0, net10.0] + # The target frameworks to build and run tests for on Windows, for the + # primary test configurations (local SQL Server 2025 and Azure SQL). + # + # These configurations carry the broadest coverage, so newer runtimes are + # validated here first before being enabled across every configuration. + # + # Note: The driver does not ship a net10.0 target framework, so net10.0 test + # assemblies resolve the net9.0 driver build. Including net10.0 here + # validates the driver running on the .NET 10 runtime. + # + - name: primaryTargetFrameworks + type: object + default: [net462, net8.0, net9.0, net10.0] + + # The target frameworks to build and run tests for on Unix, for the primary + # test configurations (local SQL Server 2025 and Azure SQL). + - name: primaryTargetFrameworksUnix + type: object + default: [net8.0, net9.0, net10.0] + # Netcore Version for Test Utilities - name: netcoreVersionTestUtils type: object @@ -412,7 +432,7 @@ stages: pool: ${{parameters.defaultPoolName }} images: Win25_Sql25: ADO-MMS25-SQL25 - TargetFrameworks: ${{parameters.targetFrameworks }} + TargetFrameworks: ${{parameters.primaryTargetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} testSets: ${{parameters.testSets }} @@ -485,7 +505,7 @@ stages: images: Win22_Azure_Sql: ADO-MMS22-SQL19 Win11_Azure_Sql: ADO-CI-Win11 - TargetFrameworks: ${{parameters.targetFrameworks }} + TargetFrameworks: ${{parameters.primaryTargetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} testSets: ${{parameters.testSets }} @@ -542,7 +562,7 @@ stages: pool: ${{parameters.defaultPoolName }} images: Ubuntu24_Sql25: ADO-UB24-SQL25 - TargetFrameworks: ${{parameters.targetFrameworksUnix }} + TargetFrameworks: ${{parameters.primaryTargetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] testSets: ${{parameters.testSets }} @@ -565,7 +585,7 @@ stages: pool: ${{parameters.defaultPoolName }} images: Ubuntu24_Azure_Sql: ADO-UB24-SQL25 - TargetFrameworks: ${{parameters.targetFrameworksUnix }} + TargetFrameworks: ${{parameters.primaryTargetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] testSets: ${{parameters.testSets }} diff --git a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml index c7fa90860f..74fa022a6c 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml @@ -175,6 +175,11 @@ extends: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + # Keep the primary (SQL 2025 / Azure SQL) configurations on the same set of + # target frameworks as everything else. Only the CI-SqlClient pipeline + # opts in to the broader .NET 10.0 coverage on those configurations. + primaryTargetFrameworks: ${{ parameters.targetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} diff --git a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml index 54a1b0e639..ce4db1ce3c 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml @@ -113,8 +113,9 @@ parameters: # These are _not_ the target frameworks to build the driver packages for. # # Note: We are excluding .NET 10.0 here to avoid consuming too many resources - # during PR pipeline runs, and until we update our 1ES images to include - # Visual Studio 2026 (18.0) whose MSBuild SDK supports .NET 10. + # across every SQL Server image. .NET 10.0 coverage is provided by the + # primaryTargetFrameworks parameter below, which applies to the SQL Server + # 2025 and Azure SQL configurations. # - name: targetFrameworks displayName: Target Frameworks on Windows @@ -126,14 +127,38 @@ parameters: # These are _not_ the target frameworks to build the driver packages for. # # Note: We are excluding .NET 10.0 here to avoid consuming too many resources - # during PR pipeline runs, and until we update our 1ES images to include - # Visual Studio 2026 (18.0) whose MSBuild SDK supports .NET 10. + # across every SQL Server image. .NET 10.0 coverage is provided by the + # primaryTargetFrameworksUnix parameter below, which applies to the SQL + # Server 2025 and Azure SQL configurations. # - name: targetFrameworksUnix displayName: Target Frameworks on Unix type: object default: [net8.0, net9.0] + # The target frameworks used by the primary test configurations (local SQL + # Server 2025 and Azure SQL) on Windows. + # + # net10.0 is included here so that unit, functional, and manual tests get + # .NET 10 coverage without paying for it on every legacy SQL Server image. + # + # Note: The driver itself does not ship a net10.0 target framework, so the + # net10.0 test assemblies resolve the net9.0 driver build. These jobs + # therefore validate the driver running on the .NET 10 runtime, rather than + # a net10.0 build of the driver. + # + - name: primaryTargetFrameworks + displayName: Target Frameworks on Windows (SQL 2025 and Azure SQL) + type: object + default: [net462, net8.0, net9.0, net10.0] + + # The target frameworks used by the primary test configurations (local SQL + # Server 2025 and Azure SQL) on Unix. + - name: primaryTargetFrameworksUnix + displayName: Target Frameworks on Unix (SQL 2025 and Azure SQL) + type: object + default: [net8.0, net9.0, net10.0] + # The timeout, in minutes, for each test job. - name: testJobTimeout displayName: Test job timeout (in minutes) @@ -175,6 +200,8 @@ extends: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + primaryTargetFrameworks: ${{ parameters.primaryTargetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.primaryTargetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} diff --git a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml index cb25fe1f4e..abb0ef5f39 100644 --- a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml @@ -135,6 +135,11 @@ extends: debug: ${{ parameters.debug }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + # Keep the primary (SQL 2025 / Azure SQL) configurations on the same set of + # target frameworks as everything else. Only the CI-SqlClient pipeline + # opts in to the broader .NET 10.0 coverage on those configurations. + primaryTargetFrameworks: ${{ parameters.targetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} diff --git a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml index b2a5edf4e4..f9578e6548 100644 --- a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml @@ -135,6 +135,11 @@ extends: debug: ${{ parameters.debug }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + # Keep the primary (SQL 2025 / Azure SQL) configurations on the same set of + # target frameworks as everything else. Only the CI-SqlClient pipeline + # opts in to the broader .NET 10.0 coverage on those configurations. + primaryTargetFrameworks: ${{ parameters.targetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} From bdc6546ee29be4150c3b6798aa064bbbe2696edd Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:06:16 -0300 Subject: [PATCH 10/51] Trigger Kerberos CI from package pipeline (#4499) * Trigger Kerberos tests from package pipeline * Restructure Kerberos and MI test pipelines * Avoid tag fetches in upstream alignment checkout (#4499) * Use current images for MI and Kerberos tests * Address PR 4499 pipeline review feedback * Remove Kerberos code coverage stage * Secure Kerberos teardown credentials (#4499) * Fix downstream package version resolution --- .../ci/kerberos/build-and-test-steps.yml | 138 -------- eng/pipelines/ci/kerberos/linux-init-step.yml | 117 ------ .../ci/kerberos/linux-setup-step.yml | 110 ++++++ ...eanup-step.yml => linux-teardown-step.yml} | 11 +- .../ci/kerberos/sqlclient-ci-kerberos-job.yml | 132 +++++++ .../sqlclient-ci-kerberos-pipeline.yml | 332 ++++-------------- .../kerberos/sqlclient-ci-kerberos-stages.yml | 107 ++++++ .../ci/kerberos/windows-setup-step.yml | 52 +++ .../sqlclient-ci-managed-instance-job.yml | 7 - ...sqlclient-ci-managed-instance-pipeline.yml | 13 +- .../sqlclient-ci-managed-instance-stage.yml | 111 ------ .../sqlclient-ci-managed-instance-stages.yml | 100 ++++++ .../ci/stress/sqlclient-ci-stress-job.yml | 10 +- .../stress/sqlclient-ci-stress-pipeline.yml | 10 +- .../steps/align-source-with-upstream-step.yml | 8 + .../steps/download-driver-packages-step.yml | 27 +- .../common/steps/download-driver-packages.ps1 | 76 ++-- 17 files changed, 633 insertions(+), 728 deletions(-) delete mode 100644 eng/pipelines/ci/kerberos/build-and-test-steps.yml delete mode 100644 eng/pipelines/ci/kerberos/linux-init-step.yml create mode 100644 eng/pipelines/ci/kerberos/linux-setup-step.yml rename eng/pipelines/ci/kerberos/{linux-cleanup-step.yml => linux-teardown-step.yml} (77%) create mode 100644 eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml create mode 100644 eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml create mode 100644 eng/pipelines/ci/kerberos/windows-setup-step.yml delete mode 100644 eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml create mode 100644 eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml diff --git a/eng/pipelines/ci/kerberos/build-and-test-steps.yml b/eng/pipelines/ci/kerberos/build-and-test-steps.yml deleted file mode 100644 index dbb3b5e1e2..0000000000 --- a/eng/pipelines/ci/kerberos/build-and-test-steps.yml +++ /dev/null @@ -1,138 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# Shared build-and-test steps used by both the Windows and Linux Kerberos jobs. -# -# Parameters: -# testFramework — The TFM to test against (e.g. net9.0, net462). -# testRunTitle — Title for the published test results (displayed in the ADO -# Tests tab). -# artifactName — Name of the published pipeline artifact that carries the -# test results and coverage files. - -parameters: - - # TFM to pass to the test targets (-p:TestFramework). - - name: testFramework - type: string - - # Title shown in the ADO Tests tab for this run. - - name: testRunTitle - type: string - - # Pipeline artifact name for test results and coverage. - - name: artifactName - type: string - -steps: - - # --------------------------------------------------------------------------- - # Build - # --------------------------------------------------------------------------- - - # Build SqlClient. - # - # The test stages build as part of their targets, but this separate step isolates build failures - # so we can fail fast before running tests. Retries are enabled intentionally (1 attempt for - # build, 2 attempts for test steps) to reduce transient infrastructure-related failures. - # - - task: DotNetCoreCLI@2 - displayName: Build SqlClient - retryCountOnTaskFailure: 1 - inputs: - command: build - projects: build.proj - arguments: >- - -t:BuildSqlClient - -p:Configuration=Release - - # --------------------------------------------------------------------------- - # Run tests in separate steps to permit focused retries. - # --------------------------------------------------------------------------- - - # Run the Unit Test suite. - - task: DotNetCoreCLI@2 - displayName: Run Unit Tests (${{ parameters.testFramework }}) - retryCountOnTaskFailure: 2 - inputs: - command: build - projects: build.proj - arguments: >- - -t:TestSqlClientUnit - -p:TestFramework=${{ parameters.testFramework }} - -p:Configuration=Release - - # Run the Functional Test suite. - - task: DotNetCoreCLI@2 - displayName: Run Functional Tests (${{ parameters.testFramework }}) - retryCountOnTaskFailure: 2 - inputs: - command: build - projects: build.proj - arguments: >- - -t:TestSqlClientFunctional - -p:TestFramework=${{ parameters.testFramework }} - -p:Configuration=Release - - # Run the Manual Test suite. - - task: DotNetCoreCLI@2 - displayName: Run Manual Tests (${{ parameters.testFramework }}) - retryCountOnTaskFailure: 2 - inputs: - command: build - projects: build.proj - arguments: >- - -t:TestSqlClientManual - -p:TestFramework=${{ parameters.testFramework }} - -p:Configuration=Release - - # --------------------------------------------------------------------------- - # Publish results & coverage - # --------------------------------------------------------------------------- - - # Publish the TRX test results to the pipeline run. - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - inputs: - testResultsFormat: VSTest - # build.proj defines TestResultsFolderPath which defaults to - # $(Build.SourcesDirectory)/test_results, so we look there for the results and coverage files. - testResultsFiles: $(Build.SourcesDirectory)/test_results/**/*.trx - mergeTestResults: true - testRunTitle: ${{ parameters.testRunTitle }} - buildConfiguration: Release - - # Azure Pipelines task conditions do not support path existence checks directly, - # so compute this once and gate later steps on the variable. - - pwsh: | - $resultsDir = "$(Build.SourcesDirectory)/test_results" - if (Test-Path -LiteralPath $resultsDir) { - Write-Host "##vso[task.setvariable variable=HasTestResultsDir]true" - } - else { - Write-Host "##vso[task.setvariable variable=HasTestResultsDir]false" - } - displayName: Detect test_results directory - condition: succeededOrFailed() - - # Give our coverage files a unique name to make it clear where they originated when we download - # the artifacts from all jobs in the merge stage. - - pwsh: | - cd $(Build.SourcesDirectory)/test_results - Get-ChildItem -Filter "*.coverage" -Recurse | - Rename-Item -NewName { "${{ parameters.testFramework }}" + $_.Name } - displayName: Rename coverage files - condition: and(succeededOrFailed(), eq(variables['HasTestResultsDir'], 'true')) - - # Publish TRX test results and coverage files as pipeline artifacts. The merge stage needs the - # coverage files from all of the jobs. - - task: PublishPipelineArtifact@1 - displayName: Publish Test Artifacts - condition: and(succeededOrFailed(), eq(variables['HasTestResultsDir'], 'true')) - inputs: - targetPath: $(Build.SourcesDirectory)/test_results - artifact: ${{ parameters.artifactName }} diff --git a/eng/pipelines/ci/kerberos/linux-init-step.yml b/eng/pipelines/ci/kerberos/linux-init-step.yml deleted file mode 100644 index c0c4603232..0000000000 --- a/eng/pipelines/ci/kerberos/linux-init-step.yml +++ /dev/null @@ -1,117 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# This template joins a Linux agent to an Active Directory domain using Kerberos -# and acquires a TGT (Ticket-Granting Ticket) for the specified domain user. -# -# Prerequisites: -# - The agent must be running on Ubuntu/Debian (uses apt-get). -# - The domain controller must be reachable from the agent network. -# -# After this step completes successfully, the agent will have: -# - Kerberos packages installed (krb5-user, realmd, sssd, adcli, etc.) -# - Hostname set to FQDN within the domain -# - NTP synchronized with the domain controller -# - Machine joined to the AD domain -# - A valid Kerberos TGT for the specified user - -parameters: - - # The Active Directory domain to join (e.g. mydomain.contoso.com). - - name: kerberosDomain - type: string - - # The Organizational Unit in which to place the computer account. - - name: kerberosDomainOU - type: string - - # The domain user account to authenticate with (sAMAccountName, without @realm). - - name: kerberosDomainUser - type: string - - # The password for the domain user account. - - name: kerberosDomainPassword - type: string - -steps: - - - bash: | - set -euo pipefail - - DOMAIN="${{ parameters.kerberosDomain }}" - DOMAIN_OU="${{ parameters.kerberosDomainOU }}" - DOMAIN_USER="${{ parameters.kerberosDomainUser }}" - DOMAIN_PASSWORD="${{ parameters.kerberosDomainPassword }}" - DOMAIN_UPPER=$(echo "$DOMAIN" | tr '[:lower:]' '[:upper:]') - - echo "Domain: $DOMAIN" - echo "Realm: $DOMAIN_UPPER" - echo "User: $DOMAIN_USER" - echo "OU: $DOMAIN_OU" - - if [ -z "$DOMAIN_PASSWORD" ]; then - echo "##vso[task.logissue type=error]KerberosDomainPassword is empty" - exit 1 - fi - - # ----------------------------------------------------------------------- - # Install Kerberos and AD integration packages - # ----------------------------------------------------------------------- - echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections - - sudo apt-get -y update - sudo apt-get install -y dialog apt-utils - sudo apt-get install -y \ - krb5-user samba sssd sssd-tools libnss-sss libpam-sss \ - ntp ntpdate realmd adcli - - # ----------------------------------------------------------------------- - # Set the hostname to FQDN within the domain - # ----------------------------------------------------------------------- - CURRENT_HOSTNAME="$(hostname)" - if [ "$CURRENT_HOSTNAME" = "$DOMAIN" ] || [[ "$CURRENT_HOSTNAME" == *".$DOMAIN" ]]; then - echo "Hostname already uses domain suffix '.$DOMAIN': $CURRENT_HOSTNAME" - else - sudo hostnamectl set-hostname "$CURRENT_HOSTNAME.$DOMAIN" - fi - - # ----------------------------------------------------------------------- - # Synchronize time with the domain controller (required for Kerberos) - # ----------------------------------------------------------------------- - if ! sudo grep -Fqx "server $DOMAIN" /etc/ntp.conf; then - echo "server $DOMAIN" | sudo tee -a /etc/ntp.conf - fi - sudo systemctl stop ntp - sudo ntpdate "$DOMAIN" - sudo systemctl start ntp - - # ----------------------------------------------------------------------- - # Configure Kerberos realm - # ----------------------------------------------------------------------- - echo "[libdefaults] - default_realm = $DOMAIN_UPPER - rdns = false" | sudo tee /etc/krb5.conf - - # ----------------------------------------------------------------------- - # Discover and join the domain - # ----------------------------------------------------------------------- - sudo realm discover "$DOMAIN_UPPER" - - echo "$DOMAIN_PASSWORD" | sudo realm join --verbose "$DOMAIN_UPPER" \ - -U "$DOMAIN_USER@$DOMAIN_UPPER" \ - --computer-ou "OU=$DOMAIN_OU" - - realm list - - # ----------------------------------------------------------------------- - # Acquire a Kerberos TGT - # ----------------------------------------------------------------------- - echo "$DOMAIN_PASSWORD" | kinit "$DOMAIN_USER@$DOMAIN_UPPER" - - klist - sudo ip addr - sudo ip route - displayName: Initialize Kerberos (domain join + kinit) diff --git a/eng/pipelines/ci/kerberos/linux-setup-step.yml b/eng/pipelines/ci/kerberos/linux-setup-step.yml new file mode 100644 index 0000000000..c5de26b3ba --- /dev/null +++ b/eng/pipelines/ci/kerberos/linux-setup-step.yml @@ -0,0 +1,110 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Configures a Linux agent, joins it to the domain, acquires a Kerberos ticket, and verifies SQL +# connectivity before running integration tests. + +parameters: + + - name: kerberosDomain + type: string + + - name: kerberosDomainOU + type: string + + - name: kerberosDomainUser + type: string + + - name: kerberosDomainPassword + type: string + +steps: + + - pwsh: | + $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json + foreach ($p in $jdata) { + $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING + $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING + $p.SupportsIntegratedSecurity = $true + } + $jdata | Add-Member -NotePropertyName "KerberosDomainUser" -NotePropertyValue $env:KERBEROS_DOMAIN_USER -Force + $jdata | Add-Member -NotePropertyName "KerberosDomainPassword" -NotePropertyValue $env:KERBEROS_DOMAIN_PASSWORD -Force + $jdata | ConvertTo-Json | Set-Content "config.jsonc" + workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities + displayName: Update test config.jsonc (Kerberos) + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) + REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) + KERBEROS_DOMAIN_USER: ${{ parameters.kerberosDomainUser }} + KERBEROS_DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} + + - bash: | + set -euo pipefail + + DOMAIN="${{ parameters.kerberosDomain }}" + DOMAIN_OU="${{ parameters.kerberosDomainOU }}" + DOMAIN_USER="${{ parameters.kerberosDomainUser }}" + DOMAIN_UPPER=$(echo "$DOMAIN" | tr '[:lower:]' '[:upper:]') + + echo "Domain: $DOMAIN" + echo "Realm: $DOMAIN_UPPER" + echo "User: $DOMAIN_USER" + echo "OU: $DOMAIN_OU" + + if [ -z "${DOMAIN_PASSWORD:-}" ]; then + echo "##vso[task.logissue type=error]KerberosDomainPassword is empty" + exit 1 + fi + + echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + + sudo apt-get -y update + sudo apt-get install -y dialog apt-utils + sudo apt-get install -y \ + krb5-user samba sssd sssd-tools libnss-sss libpam-sss \ + ntp ntpdate realmd adcli + + CURRENT_HOSTNAME="$(hostname)" + if [ "$CURRENT_HOSTNAME" = "$DOMAIN" ] || [[ "$CURRENT_HOSTNAME" == *".$DOMAIN" ]]; then + echo "Hostname already uses domain suffix '.$DOMAIN': $CURRENT_HOSTNAME" + else + sudo hostnamectl set-hostname "$CURRENT_HOSTNAME.$DOMAIN" + fi + + if ! sudo grep -Fqx "server $DOMAIN" /etc/ntp.conf; then + echo "server $DOMAIN" | sudo tee -a /etc/ntp.conf + fi + sudo systemctl stop ntp + sudo ntpdate "$DOMAIN" + sudo systemctl start ntp + + echo "[libdefaults] + default_realm = $DOMAIN_UPPER + rdns = false" | sudo tee /etc/krb5.conf + + sudo realm discover "$DOMAIN_UPPER" + + echo "$DOMAIN_PASSWORD" | sudo realm join --verbose "$DOMAIN_UPPER" \ + -U "$DOMAIN_USER@$DOMAIN_UPPER" \ + --computer-ou "OU=$DOMAIN_OU" + + realm list + + echo "$DOMAIN_PASSWORD" | kinit "$DOMAIN_USER@$DOMAIN_UPPER" + + klist + sudo ip addr + sudo ip route + displayName: Initialize Kerberos (domain join + kinit) + env: + DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} + + - pwsh: | + Install-Module -Name SqlServer -Force -Confirm:$false + Import-Module SqlServer + Invoke-Sqlcmd -Query "SELECT @@VERSION, @@SERVERNAME" -ConnectionString $env:REMOTE_TCP_CONN_STRING + displayName: Verify SQL connectivity + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) \ No newline at end of file diff --git a/eng/pipelines/ci/kerberos/linux-cleanup-step.yml b/eng/pipelines/ci/kerberos/linux-teardown-step.yml similarity index 77% rename from eng/pipelines/ci/kerberos/linux-cleanup-step.yml rename to eng/pipelines/ci/kerberos/linux-teardown-step.yml index d3491bb337..df01baffc0 100644 --- a/eng/pipelines/ci/kerberos/linux-cleanup-step.yml +++ b/eng/pipelines/ci/kerberos/linux-teardown-step.yml @@ -4,9 +4,9 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This template leaves the Active Directory domain and destroys Kerberos -# credentials. It should be referenced at the end of any job that called -# linux-init-step.yml. +# This template tears down the Linux Kerberos environment by leaving the Active Directory domain +# and destroying credentials. It should be referenced at the end of any job that called +# linux-setup-step.yml. # # All steps use condition: always() so that cleanup runs even when previous # steps fail. @@ -32,14 +32,15 @@ steps: DOMAIN="${{ parameters.kerberosDomain }}" DOMAIN_USER="${{ parameters.kerberosDomainUser }}" - DOMAIN_PASSWORD="${{ parameters.kerberosDomainPassword }}" DOMAIN_UPPER=$(echo "$DOMAIN" | tr '[:lower:]' '[:upper:]') # Leave the domain - echo "$DOMAIN_PASSWORD" | sudo realm leave "$DOMAIN_UPPER" --verbose \ + echo "${DOMAIN_PASSWORD:-}" | sudo realm leave "$DOMAIN_UPPER" --verbose \ -U "$DOMAIN_USER@$DOMAIN_UPPER" || true # Destroy the TGT and credential cache kdestroy || true displayName: Clean up Kerberos (domain leave + kdestroy) condition: always() + env: + DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml new file mode 100644 index 0000000000..f6a1b680e7 --- /dev/null +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml @@ -0,0 +1,132 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Runs the Kerberos integration tests for one OS, runtime, and SNI configuration using the exact +# packages produced by the triggering sqlclient-ci-package pipeline. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: displayName + type: string + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: jobNameSuffix + type: string + + - name: operatingSystem + type: string + values: + - Linux + - Windows + + - name: poolName + type: string + + - name: runtime + type: string + + - name: useManagedSNI + type: boolean + default: false + + - name: vmImage + type: string + +jobs: + - job: kerberos_tests_job_${{ parameters.jobNameSuffix }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 90 + + workspace: + clean: all + + pool: + name: ${{ parameters.poolName }} + demands: + - ImageOverride -equals ${{ parameters.vmImage }} + + steps: + + # Align source with the commit that produced the upstream packages while retaining the + # pipeline definitions from the commit at which this run was queued. + - template: /eng/pipelines/common/steps/align-source-with-upstream-step.yml@self + + - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self + + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + parameters: + debug: ${{ parameters.debug }} + runtimes: [8.x, 9.x, 10.x] + + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self + + - task: NuGetAuthenticate@1 + displayName: Authenticate NuGet feeds + + - ${{ if eq(parameters.operatingSystem, 'Windows') }}: + - template: /eng/pipelines/ci/kerberos/windows-setup-step.yml@self + parameters: + useManagedSNI: ${{ parameters.useManagedSNI }} + + - ${{ if eq(parameters.operatingSystem, 'Linux') }}: + - template: /eng/pipelines/ci/kerberos/linux-setup-step.yml@self + parameters: + kerberosDomain: $(KerberosDomain) + kerberosDomainOU: $(KerberosDomainOU) + kerberosDomainUser: $(KerberosDomainUser) + kerberosDomainPassword: $(KerberosDomainPassword) + + # UnitTests and FunctionalTests cover environment-independent SPN, SSPI, and connection + # string behavior in normal CI. Run only ManualTests that use the Kerberos environment. + - task: DotNetCoreCLI@2 + displayName: Run Kerberos Integration Tests + retryCountOnTaskFailure: 2 + inputs: + command: build + projects: build.proj + arguments: >- + --verbosity ${{ parameters.dotnetVerbosity }} + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.runtime }} + -p:TestSet=3 + -p:TestFilters="category!=failing&category!=flaky&category!=interactive&(FullyQualifiedName~KerberosTests|FullyQualifiedName~IntegratedAuthenticationTest|FullyQualifiedName~InstanceNameTest)" + -p:ReferenceType=Package + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=$(sqlClientPackageVersion) + -p:PackageVersionSqlServer=$(sqlServerPackageVersion) + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFormat: VSTest + testResultsFiles: $(Build.SourcesDirectory)/test_results/**/*.trx + mergeTestResults: true + testRunTitle: ${{ parameters.displayName }} + buildConfiguration: ${{ parameters.buildConfiguration }} + + - ${{ if eq(parameters.operatingSystem, 'Linux') }}: + - template: /eng/pipelines/ci/kerberos/linux-teardown-step.yml@self + parameters: + kerberosDomain: $(KerberosDomain) + kerberosDomainUser: $(KerberosDomainUser) + kerberosDomainPassword: $(KerberosDomainPassword) \ No newline at end of file diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml index 6bdea0157f..28ce2a2ed6 100644 --- a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml @@ -1,55 +1,63 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### -# ============================================================================= -# sqlclient-ci-kerberos-pipeline -# ============================================================================= -# Daily Kerberos authentication test pipeline for Microsoft.Data.SqlClient. -# Replaces the Classic "Test-SqlClient-Kerberos-Azure" pipeline. +# Kerberos authentication integration tests for Microsoft.Data.SqlClient. The tests consume the +# packages produced by sqlclient-ci-package and run on domain-connected Windows and Linux agents. # -# Schedule: daily at 07:00 UTC on internal/main. -# -# Job breakdown (10 total): -# Stage: windows → 7 jobs (net462 + net8/9/10 × NativeSNI/ManagedSNI) -# Stage: linux → 3 jobs (net8, net9, net10 — ManagedSNI only) -# Stage: Merge-Code-Coverage → 1 job -# -# Shared steps: -# Build and test steps are defined in build-and-test-steps.yml and reused -# by both stages. Each stage passes its OS-specific build target and the -# per-job testFramework matrix variable. -# -# Required ADO variable groups (link in pipeline UI): -# - kv-sqldrivers-shared (provides the agent-at-sqldrv-ad secret) -# -# Variables defined in this file (no pipeline UI configuration needed): -# - KerberosDomain sqldrv.ad -# - KerberosDomainOU agents -# - KerberosDomainUser agent -# - KerberosDomainPassword $(agent-at-sqldrv-ad) from kv-sqldrivers-shared -# - REMOTE_TCP_CONN_STRING TCP connection string to sqldrv-sql22 -# - REMOTE_NP_CONN_STRING Named Pipe connection string to sqldrv-sql22 -# ============================================================================= - -trigger: none # Scheduled runs only — no CI trigger -pr: none # Not triggered by PRs - -schedules: - - cron: '0 7 * * *' - displayName: Daily run (07:00 UTC) - branches: - include: - - internal/main - always: true +# Job breakdown: +# - Windows: net462 with native SNI, plus net8/9/10 with native and managed SNI (7 jobs). +# - Linux: net8/9/10 with managed SNI (3 jobs). name: $(date:yyyyMMdd)$(rev:.r) +# Do not trigger this pipeline for PRs or commits. +trigger: none +pr: none + +# The resource branch filter limits completion triggers to the intended upstream branch. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. +resources: + pipelines: + - pipeline: sqlclient-ci-package + source: sqlclient-ci-package + trigger: + branches: + include: + - internal/main + +parameters: + + - name: buildConfiguration + displayName: Test Build Configuration + type: string + default: Release + values: + - Debug + - Release + + - name: debug + displayName: Enable debug output + type: boolean + default: false + + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + variables: - # Our Kerberos environment doesn't change often, so we can define these variables here rather than - # in the Pipeline UI or in Libraries. + + # Kerberos environment settings shared by all jobs. KerberosDomainPassword resolves from the + # kv-sqldrivers-shared variable group imported by each OS stage. - name: KerberosDomain value: sqldrv.ad @@ -60,7 +68,7 @@ variables: value: agent - name: KerberosDomainPassword - value: $(agent-at-sqldrv-ad) # Secret from kv-sqldrivers-shared variable group + value: $(agent-at-sqldrv-ad) - name: REMOTE_TCP_CONN_STRING value: Data Source=tcp:sqldrv-sql22.sqldrv.ad\sql2022;Initial Catalog=Northwind;Integrated Security=true;Encrypt=false;TrustServerCertificate=true @@ -68,229 +76,9 @@ variables: - name: REMOTE_NP_CONN_STRING value: Data Source=np:sqldrv-sql22.sqldrv.ad\sql2022;Initial Catalog=Northwind;Integrated Security=true;Encrypt=false;TrustServerCertificate=true -# ============================================================================= -# STAGES -# ============================================================================= stages: - - # =========================================================================== - # Stage 1 - Windows (7 jobs = net462 + 3 TFs × 2 ManagedSNI) - # =========================================================================== - - stage: windows - displayName: Windows - dependsOn: [] - variables: - # KerberosDomainPassword expands at runtime in this stage from the agent-at-sqldrv-ad secret - # exposed by this variable group. - - group: kv-sqldrivers-shared - jobs: - - job: windows - displayName: Windows - timeoutInMinutes: 90 - workspace: - clean: all # Purge obj/artifacts from prior runs on self-hosted agents - strategy: - matrix: - # Azure Pipelines exposes matrix variables as environment variables for each step. - # Do not use the name targetFramework here: MSBuild imports environment variables as - # properties, and TargetFramework would leak into dotnet build build.proj, forcing - # transitive project references onto an invalid TFM (for example SqlServer.Server -> net9.0). - net462_NativeSNI: - testFramework: net462 - managedSNI: 'false' - net8_NativeSNI: - testFramework: net8.0 - managedSNI: 'false' - net8_ManagedSNI: - testFramework: net8.0 - managedSNI: 'true' - net9_NativeSNI: - testFramework: net9.0 - managedSNI: 'false' - net9_ManagedSNI: - testFramework: net9.0 - managedSNI: 'true' - net10_NativeSNI: - testFramework: net10.0 - managedSNI: 'false' - net10_ManagedSNI: - testFramework: net10.0 - managedSNI: 'true' - pool: - name: ADO-Trusted-Domain-Win-WestUS2 - demands: - - ImageOverride -equals ADO-MMS22-SQL19 - steps: - - - checkout: self - clean: true - fetchDepth: 1 - fetchTags: false - - - template: /eng/pipelines/common/steps/install-dotnet.yml@self - parameters: - runtimes: [8.x, 9.x] - - # Restore dotnet local tools (pwsh, apicompat, etc.). Required by build.proj targets - # such as _CheckPwshToolRestored that run during the SqlClient ref project build. - - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - - # --- Update test configuration --- - # Uses runtime variables from the matrix ($(managedSNI)) so we use - # inline PowerShell instead of the shared config template which - # requires compile-time parameters. - - pwsh: | - $managedSni = [System.Convert]::ToBoolean($env:MANAGED_SNI) - $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json - foreach ($p in $jdata) { - $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING - $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING - $p.SupportsIntegratedSecurity = $true - $p.UseManagedSNIOnWindows = $managedSni - } - $jdata | ConvertTo-Json | Set-Content "config.jsonc" - workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities - displayName: Update test config.jsonc - env: - REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) - REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) - MANAGED_SNI: $(managedSNI) - - # --- Prepare Windows services --- - - powershell: | - $svc = Get-Service -Name SQLBrowser -ErrorAction SilentlyContinue - if ($null -ne $svc) { - Set-Service -StartupType Automatic SQLBrowser - if ($svc.Status -ne 'Running') { Start-Service SQLBrowser } - Get-Service SQLBrowser | Select-Object Name, StartType, Status - } - displayName: Start SQL Server Browser - - - powershell: | - Set-DtcNetworkSetting -DtcName "Local" ` - -InboundTransactionsEnabled $true ` - -OutboundTransactionsEnabled $true ` - -RemoteClientAccessEnabled $true ` - -Confirm:$false - - Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True - Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC-EPMAP)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True - Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-Out)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True - Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-In)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True - displayName: Enable Network DTC Access - - # --- Build and test --- - - template: /eng/pipelines/ci/kerberos/build-and-test-steps.yml@self - parameters: - testFramework: $(testFramework) - testRunTitle: Windows-$(testFramework)-ManagedSNI_$(managedSNI) - artifactName: $(testFramework)-ManagedSNI_$(managedSNI)-$(System.JobId) - - # =========================================================================== - # Stage 2 - Linux - .NET Core + Kerberos (3 jobs = 3 TFs) - # =========================================================================== - - stage: linux - displayName: Linux - dependsOn: [] - variables: - # KerberosDomainPassword expands at runtime in this stage from the agent-at-sqldrv-ad secret - # exposed by this variable group. - - group: kv-sqldrivers-shared - jobs: - - job: linux - displayName: Linux - timeoutInMinutes: 90 - workspace: - clean: all # Purge leftovers on self-hosted agents to reduce cross-run flakiness - strategy: - matrix: - net8: - testFramework: net8.0 - net9: - testFramework: net9.0 - net10: - testFramework: net10.0 - pool: - name: ADO-Trusted-Linux-WestUS2 - demands: - - ImageOverride -equals ADO-UB20-SQL22 - - steps: - - - checkout: self - clean: true - fetchDepth: 1 - fetchTags: false - - # --- Install .NET SDK and runtimes --- - - template: /eng/pipelines/common/steps/install-dotnet.yml@self - parameters: - runtimes: [8.x, 9.x] - - # Restore dotnet local tools (pwsh, apicompat, etc.). Required by build.proj targets - # such as _CheckPwshToolRestored that run during the SqlClient ref project build. - - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - - # --- Update test configuration (with Kerberos credentials) --- - - pwsh: | - $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json - foreach ($p in $jdata) { - $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING - $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING - $p.SupportsIntegratedSecurity = $true - } - $jdata | Add-Member -NotePropertyName "KerberosDomainUser" -NotePropertyValue $env:KERBEROS_DOMAIN_USER -Force - $jdata | Add-Member -NotePropertyName "KerberosDomainPassword" -NotePropertyValue $env:KERBEROS_DOMAIN_PASSWORD -Force - $jdata | ConvertTo-Json | Set-Content "config.jsonc" - workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities - displayName: Update test config.jsonc (Kerberos) - env: - REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) - REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) - KERBEROS_DOMAIN_USER: $(KerberosDomainUser) - KERBEROS_DOMAIN_PASSWORD: $(KerberosDomainPassword) - - # --- Kerberos domain join --- - - template: /eng/pipelines/ci/kerberos/linux-init-step.yml@self - parameters: - kerberosDomain: $(KerberosDomain) - kerberosDomainOU: $(KerberosDomainOU) - kerberosDomainUser: $(KerberosDomainUser) - kerberosDomainPassword: $(KerberosDomainPassword) - - # --- Verify SQL connectivity --- - - pwsh: | - Install-Module -Name SqlServer -Force -Confirm:$false - Import-Module SqlServer - Invoke-Sqlcmd -Query "SELECT @@VERSION, @@SERVERNAME" -ConnectionString $env:REMOTE_TCP_CONN_STRING - displayName: Verify SQL connectivity - env: - REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) - - # --- Build and test --- - - template: /eng/pipelines/ci/kerberos/build-and-test-steps.yml@self - parameters: - testFramework: $(testFramework) - testRunTitle: Linux-$(testFramework) - artifactName: $(testFramework)-linux-$(System.JobId) - - # --- Kerberos cleanup (always runs) --- - - template: /eng/pipelines/ci/kerberos/linux-cleanup-step.yml@self - parameters: - kerberosDomain: $(KerberosDomain) - kerberosDomainUser: $(KerberosDomainUser) - kerberosDomainPassword: $(KerberosDomainPassword) - - # =========================================================================== - # Stage 3 — Merge Code Coverage (1 job) - # =========================================================================== - - stage: merge - displayName: Merge Code Coverage - dependsOn: - - windows - - linux - condition: succeeded() - jobs: - - template: /eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml@self - parameters: - upload: false + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml new file mode 100644 index 0000000000..86c1b95cc9 --- /dev/null +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml @@ -0,0 +1,107 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Defines separate Windows and Linux Kerberos test stages. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: netFrameworkTestRuntimes + type: object + default: [net462] + + - name: netTestRuntimes + type: object + default: [net8.0, net9.0, net10.0] + +stages: + + - stage: windows + displayName: Windows + dependsOn: [] + variables: + - group: kv-sqldrivers-shared + jobs: + + # .NET Framework uses native SNI. + - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: false + vmImage: ADO-Win25 + + # .NET runs with both native and managed SNI. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: false + vmImage: ADO-Win25 + + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Managed SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: true + vmImage: ADO-Win25 + + - stage: linux + displayName: Linux + dependsOn: [] + variables: + - group: kv-sqldrivers-shared + jobs: + + # Managed SNI is always used on non-Windows platforms. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Linux : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} + operatingSystem: Linux + poolName: ADO-Trusted-Linux-WestUS2 + runtime: ${{ runtime }} + vmImage: ADO-UB24 diff --git a/eng/pipelines/ci/kerberos/windows-setup-step.yml b/eng/pipelines/ci/kerberos/windows-setup-step.yml new file mode 100644 index 0000000000..9a736f68b9 --- /dev/null +++ b/eng/pipelines/ci/kerberos/windows-setup-step.yml @@ -0,0 +1,52 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Configures a domain-connected Windows agent for Kerberos integration tests. + +parameters: + + - name: useManagedSNI + type: boolean + +steps: + + - pwsh: | + $managedSni = [System.Convert]::ToBoolean($env:MANAGED_SNI) + $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json + foreach ($p in $jdata) { + $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING + $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING + $p.SupportsIntegratedSecurity = $true + $p.UseManagedSNIOnWindows = $managedSni + } + $jdata | ConvertTo-Json | Set-Content "config.jsonc" + workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities + displayName: Update test config.jsonc + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) + REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) + MANAGED_SNI: '${{ parameters.useManagedSNI }}' + + - powershell: | + $svc = Get-Service -Name SQLBrowser -ErrorAction SilentlyContinue + if ($null -ne $svc) { + Set-Service -StartupType Automatic SQLBrowser + if ($svc.Status -ne 'Running') { Start-Service SQLBrowser } + Get-Service SQLBrowser | Select-Object Name, StartType, Status + } + displayName: Start SQL Server Browser + + - powershell: | + Set-DtcNetworkSetting -DtcName "Local" ` + -InboundTransactionsEnabled $true ` + -OutboundTransactionsEnabled $true ` + -RemoteClientAccessEnabled $true ` + -Confirm:$false + + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC-EPMAP)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-Out)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-In)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + displayName: Enable Network DTC Access \ No newline at end of file diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml index 68f59f00c1..0141650535 100644 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml @@ -104,12 +104,7 @@ jobs: # Download the SqlClient driver packages published by the triggering sqlclient-ci-package # pipeline, stage them into the local NuGet feed, and resolve their exact versions into the # sqlClient/sqlServer/abstractions/logging/azure PackageVersion variables. - # - # sqlServerVersionOverride pins Microsoft.SqlServer.Server to the released stable 1.0.0 to - # avoid an NU1605 downgrade against the >= 1.0.0 dependency from Microsoft.SqlServer.Types. - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self - parameters: - sqlServerVersionOverride: 1.0.0 # Install the .NET SDK and the runtimes needed to execute the test frameworks. - template: /eng/pipelines/common/steps/install-dotnet.yml@self @@ -170,8 +165,6 @@ jobs: -p:TestSet=123 -p:ReferenceType=Package -p:Configuration=${{ parameters.buildConfiguration }} - -p:PackageVersionAbstractions=$(abstractionsPackageVersion) - -p:PackageVersionLogging=$(loggingPackageVersion) -p:PackageVersionSqlClient=$(sqlClientPackageVersion) -p:PackageVersionSqlServer=$(sqlServerPackageVersion) -p:TestResultsFolderPath=TestResults diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml index d579d11994..83acbd34d0 100644 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml @@ -22,7 +22,9 @@ name: $(DayOfYear)$(Rev:rr) pr: none trigger: none -# Trigger this pipeline after successful runs of the sqlclient-ci-package pipeline. +# The resource branch filter limits completion triggers to the intended upstream branch. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. resources: pipelines: @@ -37,7 +39,10 @@ resources: # added to the project, this resource will fail to resolve and the folder path must be added. - pipeline: sqlclient-ci-package source: sqlclient-ci-package - trigger: true + trigger: + branches: + include: + - internal/main # Pipeline parameters, visible in the Azure DevOps UI. parameters: @@ -76,9 +81,9 @@ parameters: # The stages to run. stages: - # Run the Managed Instance tests. The .NET and .NET Framework runtimes are defaulted by the stage + # Run the Managed Instance tests. The .NET and .NET Framework runtimes are defaulted by the stages # template, which runs both native and managed SNI for .NET on Windows. - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml@self + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml@self parameters: buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml deleted file mode 100644 index b6a14c54a7..0000000000 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml +++ /dev/null @@ -1,111 +0,0 @@ -#################################################################################################### -# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this -# file to you under the MIT license. See the LICENSE file in the project root for more information. -#################################################################################################### - -# This stage builds and runs the SqlClient Unit, Functional, and Manual test suites against an Azure -# SQL Managed Instance, building the test projects in "Package" mode against the NuGet packages -# produced by the sqlclient-ci-package pipeline. -# -# It fans out to one job per OS: -# -# - Windows: one native-SNI job per .NET Framework runtime, plus native- and managed-SNI jobs per -# .NET runtime. -# - Linux: one job per .NET runtime (managed SNI is always used on non-Windows). -# -# This template defines a stage named 'managed_instance_tests_stage'. - -parameters: - - # The type of build to produce (Debug or Release). - - name: buildConfiguration - type: string - values: - - Debug - - Release - - # True to enable debugging steps. - - name: debug - type: boolean - - # The verbosity level for the dotnet CLI commands. - - name: dotnetVerbosity - type: string - values: - - quiet - - minimal - - normal - - detailed - - diagnostic - - # The list of .NET (core) runtimes to test against. The same set is used on every OS. - - name: netTestRuntimes - type: object - default: [net8.0, net9.0, net10.0] - - # The list of .NET Framework runtimes to test against on Windows. - - name: netFrameworkTestRuntimes - type: object - default: [net462] - -stages: - - stage: managed_instance_tests_stage - displayName: Run Managed Instance Tests - - jobs: - - # ---------------------------------------------------------------------------------------------- - # Windows: .NET Framework uses native SNI; .NET runs with both native and managed SNI. - - - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - displayName: 'Win : Native SNI : ${{ runtime }}' - jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} - operatingSystem: Windows - runtime: ${{ runtime }} - useManagedSNI: false - vmImage: ADO-MMS25-SQL25 - - - ${{ each runtime in parameters.netTestRuntimes }}: - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayName: 'Win : Native SNI : ${{ runtime }}' - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} - operatingSystem: Windows - runtime: ${{ runtime }} - useManagedSNI: false - vmImage: ADO-MMS25-SQL25 - - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayName: 'Win : Managed SNI : ${{ runtime }}' - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} - operatingSystem: Windows - runtime: ${{ runtime }} - useManagedSNI: true - vmImage: ADO-MMS25-SQL25 - - # ---------------------------------------------------------------------------------------------- - # Linux: one job per .NET runtime (managed SNI is always used on non-Windows). - - - ${{ each runtime in parameters.netTestRuntimes }}: - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayName: 'Linux : ${{ runtime }}' - jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - operatingSystem: Linux - runtime: ${{ runtime }} - vmImage: ADO-UB24-SQL25 diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml new file mode 100644 index 0000000000..e96aef1bc3 --- /dev/null +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml @@ -0,0 +1,100 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Defines separate Windows and Linux stages that run ManualTests against Azure SQL Managed Instance +# using the packages produced by sqlclient-ci-package. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: netTestRuntimes + type: object + default: [net8.0, net9.0, net10.0] + + - name: netFrameworkTestRuntimes + type: object + default: [net462] + +stages: + + - stage: windows + displayName: Windows + dependsOn: [] + jobs: + + # .NET Framework uses native SNI. + - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + displayName: 'Win : Native SNI : ${{ runtime }}' + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + runtime: ${{ runtime }} + useManagedSNI: false + vmImage: ADO-Win25 + + # .NET runs with both native and managed SNI. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + runtime: ${{ runtime }} + useManagedSNI: false + vmImage: ADO-Win25 + + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Managed SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + runtime: ${{ runtime }} + useManagedSNI: true + vmImage: ADO-Win25 + + - stage: linux + displayName: Linux + dependsOn: [] + jobs: + + # Managed SNI is always used on non-Windows platforms. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Linux : ${{ runtime }}' + jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + operatingSystem: Linux + runtime: ${{ runtime }} + vmImage: ADO-UB24 diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml index efd4f233cf..d70ce90c33 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml @@ -141,7 +141,7 @@ jobs: value: >- -p:ReferenceType=Package -p:SqlClientPackageVersion=$(sqlClientPackageVersion) - -p:AzurePackageVersion=$(azurePackageVersion) + -p:AzurePackageVersion=$(sqlClientPackageVersion) # dotnet CLI options for build. - name: dotnetBuildOpts @@ -209,14 +209,8 @@ jobs: # Download the SqlClient driver packages published by the triggering sqlclient-ci-package # pipeline, stage them into the local NuGet feed, and resolve their exact versions. The - # sqlClientPackageVersion and azurePackageVersion variables are consumed by the referenceArgs - # variable above. - # - # sqlServerVersionOverride pins Microsoft.SqlServer.Server to the released stable 1.0.0 to - # avoid an NU1605 downgrade against the >= 1.0.0 dependency from Microsoft.SqlServer.Types. + # sqlClientPackageVersion variable is consumed by the referenceArgs variable above. - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self - parameters: - sqlServerVersionOverride: 1.0.0 # Authenticate with NuGet feeds so that upstream packages (e.g. runtime host packs) can be # fetched through the ADO Artifacts feed. diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml index 74283a8128..74403bd743 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml @@ -21,7 +21,9 @@ name: $(DayOfYear)$(Rev:rr) pr: none trigger: none -# Trigger this pipeline after successful runs of the sqlclient-ci-package pipeline. +# The resource branch filters limit completion triggers to the intended upstream branches. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. # # The pipeline identifiers are displayed in the Azure DevOps UI, so it is helpful if they indicate # the project, folder, and pipeline name, hence the verbose values below. @@ -44,7 +46,11 @@ resources: # per-project approach instead of this single shared definition). - pipeline: sqlclient-ci-package source: sqlclient-ci-package - trigger: true + trigger: + branches: + include: + - main + - internal/main # Pipeline parameters, visible in the Azure DevOps UI. parameters: diff --git a/eng/pipelines/common/steps/align-source-with-upstream-step.yml b/eng/pipelines/common/steps/align-source-with-upstream-step.yml index 11ad2742b7..39249d572d 100644 --- a/eng/pipelines/common/steps/align-source-with-upstream-step.yml +++ b/eng/pipelines/common/steps/align-source-with-upstream-step.yml @@ -36,6 +36,8 @@ steps: - checkout: self clean: true fetchDepth: 0 + fetchTags: false + persistCredentials: true # Align the working-tree source with the commit that built the upstream artifacts, so the projects # compile against the matching (internal) API surface. When the upstream commit is unavailable @@ -43,8 +45,14 @@ steps: - pwsh: | $ErrorActionPreference = 'Stop' $sha = "$(resources.pipeline.${{ parameters.upstreamPipeline }}.sourceCommit)" + if ($sha -notmatch '\A[0-9a-fA-F]{40}\z') { + throw "Invalid ${{ parameters.upstreamPipeline }} commit SHA: '$sha'." + } $pipelineSourceSha = git rev-parse HEAD if ($LASTEXITCODE -ne 0) { throw "Failed to resolve the queued pipeline commit." } + Write-Host "Fetching ${{ parameters.upstreamPipeline }} commit $sha" + git fetch --no-tags origin $sha + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch upstream commit $sha." } Write-Host "Aligning source to ${{ parameters.upstreamPipeline }} commit $sha" git checkout --force $sha if ($LASTEXITCODE -ne 0) { throw "Failed to check out upstream commit $sha." } diff --git a/eng/pipelines/common/steps/download-driver-packages-step.yml b/eng/pipelines/common/steps/download-driver-packages-step.yml index aed4456d31..807affea13 100644 --- a/eng/pipelines/common/steps/download-driver-packages-step.yml +++ b/eng/pipelines/common/steps/download-driver-packages-step.yml @@ -5,14 +5,12 @@ # This template downloads the SqlClient driver packages published by the sqlclient-ci-package # pipeline, copies them into the local NuGet feed (packages/), and resolves the exact package -# versions from the .nupkg filenames. The resolved versions are exposed as job-scoped pipeline -# variables for use by downstream build/test steps: +# versions from the .nupkg filenames. Required SqlClient-family packages are validated to share +# one version. The resolved versions are exposed as job-scoped pipeline variables for use by +# downstream build/test steps: # -# sqlClientPackageVersion Microsoft.Data.SqlClient -# sqlServerPackageVersion Microsoft.SqlServer.Server -# abstractionsPackageVersion Microsoft.Data.SqlClient.Extensions.Abstractions -# loggingPackageVersion Microsoft.Data.SqlClient.Internal.Logging -# azurePackageVersion Microsoft.Data.SqlClient.Extensions.Azure +# sqlClientPackageVersion SqlClient family +# sqlServerPackageVersion Microsoft.SqlServer.Server # # The consuming pipeline MUST declare the triggering pipeline as a resource whose alias matches the # pipelineResource parameter, e.g.: @@ -42,16 +40,6 @@ parameters: type: string default: $(Build.SourcesDirectory)/packages - # Optional override for the Microsoft.SqlServer.Server version. When set, this value is used for - # the sqlServerPackageVersion variable instead of the version resolved from the artifact .nupkg - # filename. Temporary workaround: the CI 'X.Y.Z-ci' prerelease of Microsoft.SqlServer.Server is - # treated as a downgrade of the '>= X.Y.Z' stable dependency that Microsoft.SqlServer.Types pulls - # in (NU1605); passing the released stable version avoids the downgrade. Overall package - # versioning is being addressed separately. - - name: sqlServerVersionOverride - type: string - default: '' - steps: # Download the SqlClient driver packages published by the triggering pipeline into the pipeline @@ -60,8 +48,8 @@ steps: artifact: ${{ parameters.artifactName }} displayName: Download SqlClient Driver Packages - # Copy the downloaded packages into the local NuGet feed and resolve the exact package versions - # from the .nupkg filenames, exposing each as a pipeline variable. + # Copy the downloaded packages into the local NuGet feed, validate the SqlClient family version, + # and expose the two independently versioned package units as pipeline variables. - task: PowerShell@2 displayName: Stage Packages and Resolve Versions inputs: @@ -71,4 +59,3 @@ steps: arguments: >- -FeedPath "${{ parameters.feedPath }}" -ArtifactDirectory "$(Pipeline.Workspace)/${{ parameters.pipelineResource }}/${{ parameters.artifactName }}" - -SqlServerVersionOverride "${{ parameters.sqlServerVersionOverride }}" diff --git a/eng/pipelines/common/steps/download-driver-packages.ps1 b/eng/pipelines/common/steps/download-driver-packages.ps1 index 39f7a38493..4392508991 100644 --- a/eng/pipelines/common/steps/download-driver-packages.ps1 +++ b/eng/pipelines/common/steps/download-driver-packages.ps1 @@ -4,14 +4,12 @@ .DESCRIPTION Copies the .nupkg and .snupkg files downloaded from an upstream SqlClient package artifact into - a local NuGet feed. It then resolves package versions from the .nupkg filenames and emits Azure - Pipelines logging commands that create these job-scoped variables for downstream tasks: + a local NuGet feed. It resolves package versions from the .nupkg filenames, verifies that all + required SqlClient-family packages share one version, and emits Azure Pipelines logging commands + that create these job-scoped variables for downstream tasks: - sqlClientPackageVersion Microsoft.Data.SqlClient - sqlServerPackageVersion Microsoft.SqlServer.Server - abstractionsPackageVersion Microsoft.Data.SqlClient.Extensions.Abstractions - loggingPackageVersion Microsoft.Data.SqlClient.Internal.Logging - azurePackageVersion Microsoft.Data.SqlClient.Extensions.Azure + sqlClientPackageVersion SqlClient family + sqlServerPackageVersion Microsoft.SqlServer.Server Every required .nupkg must be present in ArtifactDirectory. Symbol packages are optional. The script stops on missing required packages, copy failures, and ambiguous filesystem errors. @@ -24,10 +22,6 @@ Directory containing the .nupkg files downloaded from the upstream pipeline artifact. Optional .snupkg files in this directory are copied when present. -.PARAMETER SqlServerVersionOverride - Optional version to expose as sqlServerPackageVersion instead of resolving the version from the - Microsoft.SqlServer.Server .nupkg filename. The package itself is still required and staged. - .EXAMPLE ./download-driver-packages.ps1 ` -FeedPath 'C:\agent\_work\1\s\packages' ` @@ -35,14 +29,6 @@ Stages all driver packages and resolves every version from its package filename. -.EXAMPLE - ./download-driver-packages.ps1 ` - -FeedPath '/agent/_work/1/s/packages' ` - -ArtifactDirectory '/agent/_work/1/sqlclient-ci-package/SqlClient-Driver-Packages' ` - -SqlServerVersionOverride '1.0.0' - - Stages all packages but exposes 1.0.0 as sqlServerPackageVersion. - .OUTPUTS None. Results are emitted as Azure Pipelines task.setvariable logging commands. @@ -61,9 +47,7 @@ param( [string]$FeedPath, [Parameter(Mandatory)] - [string]$ArtifactDirectory, - - [string]$SqlServerVersionOverride = '' + [string]$ArtifactDirectory ) Set-StrictMode -Version Latest @@ -94,47 +78,51 @@ function Resolve-PackageVersion { return [regex]::Match($packages[0].Name, $Pattern).Groups[1].Value } -# Patterns are anchored so the Microsoft.Data.SqlClient package does not also match extension -# packages whose IDs begin with Microsoft.Data.SqlClient. -$packages = @( +# Patterns are anchored so Microsoft.Data.SqlClient does not also match family packages whose IDs +# begin with Microsoft.Data.SqlClient. +$sqlClientFamilyPackages = @( @{ - Variable = 'sqlClientPackageVersion' Name = 'Microsoft.Data.SqlClient' Pattern = '^Microsoft\.Data\.SqlClient\.(\d[^\/]*)\.nupkg$' }, @{ - Variable = 'sqlServerPackageVersion' - Name = 'Microsoft.SqlServer.Server' - Pattern = '^Microsoft\.SqlServer\.Server\.(\d[^\/]*)\.nupkg$' - }, - @{ - Variable = 'abstractionsPackageVersion' Name = 'Microsoft.Data.SqlClient.Extensions.Abstractions' Pattern = '^Microsoft\.Data\.SqlClient\.Extensions\.Abstractions\.(\d[^\/]*)\.nupkg$' }, @{ - Variable = 'loggingPackageVersion' Name = 'Microsoft.Data.SqlClient.Internal.Logging' Pattern = '^Microsoft\.Data\.SqlClient\.Internal\.Logging\.(\d[^\/]*)\.nupkg$' }, @{ - Variable = 'azurePackageVersion' Name = 'Microsoft.Data.SqlClient.Extensions.Azure' Pattern = '^Microsoft\.Data\.SqlClient\.Extensions\.Azure\.(\d[^\/]*)\.nupkg$' + }, + @{ + Name = 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' + Pattern = '^Microsoft\.Data\.SqlClient\.AlwaysEncrypted\.AzureKeyVaultProvider\.(\d[^\/]*)\.nupkg$' } ) -foreach ($package in $packages) { - $artifactVersion = Resolve-PackageVersion $ArtifactDirectory $package.Pattern $package.Name +$sqlClientPackageVersion = Resolve-PackageVersion ` + $ArtifactDirectory ` + $sqlClientFamilyPackages[0].Pattern ` + $sqlClientFamilyPackages[0].Name - if ($package.Variable -eq 'sqlServerPackageVersion' -and - -not [string]::IsNullOrWhiteSpace($SqlServerVersionOverride)) { - $version = $SqlServerVersionOverride - Write-Host "Overriding $($package.Name) version from $artifactVersion to $version" - } else { - $version = $artifactVersion - Write-Host "Resolved $($package.Name) version: $version" +foreach ($package in $sqlClientFamilyPackages) { + $version = Resolve-PackageVersion $ArtifactDirectory $package.Pattern $package.Name + + if ($version -ne $sqlClientPackageVersion) { + throw "$($package.Name) version $version does not match SqlClient family version $sqlClientPackageVersion" } - Write-Host "##vso[task.setvariable variable=$($package.Variable)]$version" + Write-Host "Validated $($package.Name) version: $version" } + +$sqlServerPackageVersion = Resolve-PackageVersion ` + $ArtifactDirectory ` + '^Microsoft\.SqlServer\.Server\.(\d[^\/]*)\.nupkg$' ` + 'Microsoft.SqlServer.Server' + +Write-Host "Resolved Microsoft.SqlServer.Server version: $sqlServerPackageVersion" +Write-Host "##vso[task.setvariable variable=sqlClientPackageVersion]$sqlClientPackageVersion" +Write-Host "##vso[task.setvariable variable=sqlServerPackageVersion]$sqlServerPackageVersion" From 5c82580e463c8d5ea81fe4b31c84007c34799fee Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:21:38 -0700 Subject: [PATCH 11/51] Clarify Entra authority URL parsing (#4630) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../ActiveDirectoryAuthenticationProvider.cs | 59 +++++++--------- .../Azure/test/AuthorityParsingTests.cs | 70 +++++++++---------- 2 files changed, 59 insertions(+), 70 deletions(-) diff --git a/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs b/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs index 3bdce5c751..cbb0371bec 100644 --- a/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs +++ b/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs @@ -252,7 +252,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti // ("https://login.microsoftonline.com/{tenantId}/oauth2/authorize"), so the tenant is // taken from the first path segment rather than the last. - if (!TryParseAuthority(parameters.Authority, out string authorityHost, out string tenant, out string msalAuthority)) + if (!TryParseAuthority(parameters.Authority, out string authorityUrl, out string tenant)) { throw new Extensions.Azure.AuthenticationException( parameters.AuthenticationMethod, @@ -266,7 +266,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryDefault) { // Cache DefaultAzureCredential based on scope, authority host, tenant, and clientId - TokenCredentialKey tokenCredentialKey = new(typeof(DefaultAzureCredential), authorityHost, scope, tenant, clientId); + TokenCredentialKey tokenCredentialKey = new(typeof(DefaultAzureCredential), authorityUrl, scope, tenant, clientId); AccessToken accessToken = await GetTokenAsync(tokenCredentialKey, string.Empty, tokenRequestContext, cts.Token).ConfigureAwait(false); SqlClientEventSource.Log.TryTraceEvent("AcquireTokenAsync | Acquired access token for Default auth mode. Expiry Time: {0}", accessToken.ExpiresOn); return new SqlAuthenticationToken(accessToken.Token, accessToken.ExpiresOn); @@ -275,7 +275,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryManagedIdentity || parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryMSI) { // Cache ManagedIdentityCredential based on scope, authority host, and clientId - TokenCredentialKey tokenCredentialKey = new(typeof(ManagedIdentityCredential), authorityHost, scope, string.Empty, clientId); + TokenCredentialKey tokenCredentialKey = new(typeof(ManagedIdentityCredential), authorityUrl, scope, string.Empty, clientId); AccessToken accessToken = await GetTokenAsync(tokenCredentialKey, string.Empty, tokenRequestContext, cts.Token).ConfigureAwait(false); SqlClientEventSource.Log.TryTraceEvent("AcquireTokenAsync | Acquired access token for Managed Identity auth mode. Expiry Time: {0}", accessToken.ExpiresOn); return new SqlAuthenticationToken(accessToken.Token, accessToken.ExpiresOn); @@ -284,7 +284,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryServicePrincipal) { // Cache ClientSecretCredential based on scope, authority host, tenant, and clientId - TokenCredentialKey tokenCredentialKey = new(typeof(ClientSecretCredential), authorityHost, scope, tenant, clientId); + TokenCredentialKey tokenCredentialKey = new(typeof(ClientSecretCredential), authorityUrl, scope, tenant, clientId); string password = parameters.Password is null ? string.Empty : parameters.Password; AccessToken accessToken = await GetTokenAsync(tokenCredentialKey, password, tokenRequestContext, cts.Token).ConfigureAwait(false); SqlClientEventSource.Log.TryTraceEvent("AcquireTokenAsync | Acquired access token for Active Directory Service Principal auth mode. Expiry Time: {0}", accessToken.ExpiresOn); @@ -294,7 +294,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryWorkloadIdentity) { // Cache WorkloadIdentityCredential based on authority host and clientId - TokenCredentialKey tokenCredentialKey = new(typeof(WorkloadIdentityCredential), authorityHost, string.Empty, string.Empty, clientId); + TokenCredentialKey tokenCredentialKey = new(typeof(WorkloadIdentityCredential), authorityUrl, string.Empty, string.Empty, clientId); // If either tenant id, client id, or the token file path are not specified when fetching the token, // a CredentialUnavailableException will be thrown instead AccessToken accessToken = await GetTokenAsync(tokenCredentialKey, string.Empty, tokenRequestContext, cts.Token).ConfigureAwait(false); @@ -322,11 +322,12 @@ public override async Task AcquireTokenAsync(SqlAuthenti #endif : s_systemBrowserRedirectUri; + string msalAuthorityUrl = authorityUrl + tenant; PublicClientAppKey pcaKey = #if NETFRAMEWORK - new(msalAuthority, redirectUri, _applicationClientId, _iWin32WindowFunc); + new(msalAuthorityUrl, redirectUri, _applicationClientId, _iWin32WindowFunc); #else - new(msalAuthority, redirectUri, _applicationClientId); + new(msalAuthorityUrl, redirectUri, _applicationClientId); #endif AuthenticationResult? result = null; @@ -364,7 +365,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti else if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryPassword) #pragma warning restore CS0618 // Type or member is obsolete { - string pwCacheKey = GetAccountPwCacheKey(msalAuthority, parameters.UserId); + string pwCacheKey = GetAccountPwCacheKey(msalAuthorityUrl, parameters.UserId); object? previousPw = s_accountPwCache.Get(pwCacheKey); string password = parameters.Password is null ? string.Empty : parameters.Password; byte[] currPwHash = GetHash(password); @@ -547,26 +548,23 @@ or AuthenticationRequiredException } /// - /// Splits an Entra ID authority URL (the STSURL provided by the server in the FEDAUTHINFO TDS - /// token) into the authority host and the tenant. + /// Splits the STSURL provided by the server in the FEDAUTHINFO TDS token into the Entra ID + /// authority URL and tenant. /// - /// - /// The authority URL, e.g. https://login.microsoftonline.com/{tenantId}. Some services + /// + /// The STSURL, e.g. https://login.microsoftonline.com/{tenantId}. Some services /// (for example the Dataverse/Dynamics 365 TDS endpoint) return an ADAL v1 style URL such as /// https://login.microsoftonline.com/{tenantId}/oauth2/authorize. /// - /// - /// Receives the authority host with a trailing slash, e.g. https://login.microsoftonline.com/. + /// + /// Receives the authority URL with a trailing slash, e.g. https://login.microsoftonline.com/. /// /// /// Receives the tenant (the first path segment of the authority URL), which may be a tenant id, /// a domain name, or one of the common/organizations/consumers placeholders. /// - /// - /// Receives the normalized authority (host + tenant) suitable for MSAL's WithAuthority. - /// /// - /// true if the authority URL is a well-formed, absolute HTTPS URL carrying a tenant + /// true if the STSURL is a well-formed, absolute HTTPS URL carrying a tenant /// segment; otherwise false. /// /// @@ -577,17 +575,16 @@ or AuthenticationRequiredException /// /// Entra ID authorities are always absolute HTTPS URLs, so anything else is rejected rather /// than guessed at. Both MSAL (WithAuthority) and Azure.Identity - /// (TokenCredentialOptions.AuthorityHost) require an absolute URI as well, so an + /// (TokenCredentialOptions.AuthorityHost) require an absolute URL as well, so an /// unparseable authority cannot produce a working credential. /// /// internal static bool TryParseAuthority( - string authorityUrl, - out string authorityHost, - out string tenant, - out string msalAuthority) + string stsUrl, + out string authorityUrl, + out string tenant) { - if (Uri.TryCreate(authorityUrl, UriKind.Absolute, out Uri? uri) && + if (Uri.TryCreate(stsUrl, UriKind.Absolute, out Uri? uri) && uri.Scheme == Uri.UriSchemeHttps && uri.Segments.Length > 1) { @@ -598,15 +595,13 @@ internal static bool TryParseAuthority( if (tenant.Length > 0) { - authorityHost = uri.GetLeftPart(UriPartial.Authority) + "/"; - msalAuthority = authorityHost + tenant; + authorityUrl = uri.GetLeftPart(UriPartial.Authority) + "/"; return true; } } - authorityHost = string.Empty; + authorityUrl = string.Empty; tenant = string.Empty; - msalAuthority = string.Empty; return false; } @@ -840,15 +835,15 @@ private static async Task GetTokenAsync(TokenCredentialKey tokenCre /// /// Builds the cache key used to remember which password was last validated for an account. /// - /// - /// The normalized authority (host + tenant) from . The + /// + /// The normalized MSAL authority URL (authority URL + tenant). The /// normalized form is used so that two spellings of the same tenant (for example a bare /// tenant endpoint and an OAuth v1 /oauth2/authorize endpoint) share a single entry. /// /// The user id being authenticated, which may be null. - private static string GetAccountPwCacheKey(string msalAuthority, string? userId) + private static string GetAccountPwCacheKey(string msalAuthorityUrl, string? userId) { - return msalAuthority + "+" + userId; + return msalAuthorityUrl + "+" + userId; } private static byte[] GetHash(string input) diff --git a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AuthorityParsingTests.cs b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AuthorityParsingTests.cs index cbbd56e3d1..bd97f3926d 100644 --- a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AuthorityParsingTests.cs +++ b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AuthorityParsingTests.cs @@ -6,7 +6,7 @@ namespace Microsoft.Data.SqlClient.Extensions.Azure.Test; /// /// Tests for splitting the STSURL supplied by the server in the FEDAUTHINFO TDS token into an -/// authority host and a tenant. +/// authority URL and a tenant. /// /// /// The cases below only cover authority shapes that Entra ID actually documents: @@ -16,97 +16,93 @@ public class AuthorityParsingTests { private const string Tenant = "72f988bf-86f1-41af-91ab-2d7cd011db47"; - public static TheoryData AuthorityData => new() + /// + /// Provides documented STSURL shapes and their expected authority URL and tenant. + /// + public static TheoryData AuthorityData => new() { // Azure SQL / Fabric style authority. { $"https://login.microsoftonline.com/{Tenant}", "https://login.microsoftonline.com/", - Tenant, - $"https://login.microsoftonline.com/{Tenant}" + Tenant }, // Trailing slash. { $"https://login.microsoftonline.com/{Tenant}/", "https://login.microsoftonline.com/", - Tenant, - $"https://login.microsoftonline.com/{Tenant}" + Tenant }, // v1.0 authorize endpoint, as returned by the Dataverse / Dynamics 365 TDS endpoint. { $"https://login.microsoftonline.com/{Tenant}/oauth2/authorize", "https://login.microsoftonline.com/", - Tenant, - $"https://login.microsoftonline.com/{Tenant}" + Tenant }, // v2.0 token endpoint. { $"https://login.microsoftonline.com/{Tenant}/oauth2/v2.0/token", "https://login.microsoftonline.com/", - Tenant, - $"https://login.microsoftonline.com/{Tenant}" + Tenant }, // US Government cloud. { $"https://login.microsoftonline.us/{Tenant}/oauth2/authorize", "https://login.microsoftonline.us/", - Tenant, - $"https://login.microsoftonline.us/{Tenant}" + Tenant }, // Microsoft Azure operated by 21Vianet. { $"https://login.partner.microsoftonline.cn/{Tenant}", "https://login.partner.microsoftonline.cn/", - Tenant, - $"https://login.partner.microsoftonline.cn/{Tenant}" + Tenant }, // Domain-name tenant. { "https://login.microsoftonline.com/contoso.onmicrosoft.com", "https://login.microsoftonline.com/", - "contoso.onmicrosoft.com", - "https://login.microsoftonline.com/contoso.onmicrosoft.com" + "contoso.onmicrosoft.com" }, // Placeholder tenant. { "https://login.microsoftonline.com/common/oauth2/authorize", "https://login.microsoftonline.com/", - "common", - "https://login.microsoftonline.com/common" + "common" }, { "https://login.microsoftonline.com/organizations", "https://login.microsoftonline.com/", - "organizations", - "https://login.microsoftonline.com/organizations" + "organizations" }, { "https://login.microsoftonline.com/consumers", "https://login.microsoftonline.com/", - "consumers", - "https://login.microsoftonline.com/consumers" + "consumers" }, }; + /// + /// Verifies each supported STSURL shape yields the authority URL and first path segment tenant. + /// [Theory] [MemberData(nameof(AuthorityData))] - public void TryParseAuthority_SplitsHostAndTenant( - string authorityUrl, - string expectedHost, - string expectedTenant, - string expectedMsalAuthority) + public void TryParseAuthority_SplitsAuthorityUrlAndTenant( + string stsUrl, + string expectedAuthorityUrl, + string expectedTenant) { Assert.True(ActiveDirectoryAuthenticationProvider.TryParseAuthority( - authorityUrl, - out string host, - out string tenant, - out string msalAuthority)); + stsUrl, + out string authorityUrl, + out string tenant)); - Assert.Equal(expectedHost, host); + Assert.Equal(expectedAuthorityUrl, authorityUrl); Assert.Equal(expectedTenant, tenant); - Assert.Equal(expectedMsalAuthority, msalAuthority); } + /// + /// Verifies invalid STSURLs are rejected without returning partial authority data. + /// [Theory] // A tenant is required; an authority without one cannot yield a usable credential. [InlineData("https://login.microsoftonline.com")] @@ -122,13 +118,11 @@ public void TryParseAuthority_RejectsInvalidAuthority(string authorityUrl) { Assert.False(ActiveDirectoryAuthenticationProvider.TryParseAuthority( authorityUrl, - out string host, - out string tenant, - out string msalAuthority)); + out string parsedAuthorityUrl, + out string tenant)); - Assert.Equal(string.Empty, host); + Assert.Equal(string.Empty, parsedAuthorityUrl); Assert.Equal(string.Empty, tenant); - Assert.Equal(string.Empty, msalAuthority); } /// From 97d2a21b0a5802e9259fd04e54815620f74499fb Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:37:42 -0700 Subject: [PATCH 12/51] Address AccessToken authentication feedback (#4629) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Microsoft.Data.SqlClient/SqlConnection.xml | 5 +++-- .../src/Microsoft/Data/Common/AdapterUtil.cs | 7 ++----- .../Microsoft/Data/SqlClient/SqlConnection.cs | 6 +++--- .../src/Resources/Strings.Designer.cs | 15 +++------------ .../src/Resources/Strings.resx | 7 ++----- .../SimulatedServerTests/ConnectionTests.cs | 16 ++++++++++++---- 6 files changed, 25 insertions(+), 31 deletions(-) diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml index 5f88a1498e..2db0e08549 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml @@ -262,10 +262,11 @@ The following example creates a and a This property is mutually exclusive with the - + , + , and - properties, among others. Setting this property when + properties. Setting this property when is already set throws , because SSPI is an diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs index e2e73b55b1..f9410a5c39 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs @@ -1488,11 +1488,8 @@ internal static Exception InvalidMixedUsageOfAccessTokenCallbackAndAuthenticatio internal static Exception InvalidMixedUsageOfAccessTokenCallbackAndIntegratedSecurity() => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidMixedUsageOfAccessTokenCallbackAndIntegratedSecurity)); - internal static Exception InvalidMixedUsageOfAccessTokenAndSspiContextProvider() - => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidMixedUsageOfAccessTokenAndSspiContextProvider)); - - internal static Exception InvalidMixedUsageOfSspiContextProviderAndAccessToken() - => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidMixedUsageOfSspiContextProviderAndAccessToken)); + internal static Exception InvalidMixedUsageOfAccessTokenProperties() + => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidMixedUsageOfAccessTokenProperties)); #endregion internal static readonly IntPtr s_ptrZero = IntPtr.Zero; diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnection.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnection.cs index b8b693132e..7368798bb8 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnection.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnection.cs @@ -1199,7 +1199,7 @@ private void CheckAndThrowOnInvalidCombinationOfConnectionOptionAndAccessToken(S if (_sspiContextProvider != null) { - throw ADP.InvalidMixedUsageOfAccessTokenAndSspiContextProvider(); + throw ADP.InvalidMixedUsageOfAccessTokenProperties(); } } @@ -1226,7 +1226,7 @@ private void CheckAndThrowOnInvalidCombinationOfConnectionOptionAndAccessTokenCa if (_sspiContextProvider != null) { - throw ADP.InvalidMixedUsageOfAccessTokenAndSspiContextProvider(); + throw ADP.InvalidMixedUsageOfAccessTokenProperties(); } } @@ -1238,7 +1238,7 @@ private void CheckAndThrowOnInvalidCombinationOfConnectionOptionAndSspiContextPr { if (_accessToken != null || _accessTokenCallback != null) { - throw ADP.InvalidMixedUsageOfSspiContextProviderAndAccessToken(); + throw ADP.InvalidMixedUsageOfAccessTokenProperties(); } } diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs index c8f18d38bc..27327a1272 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs @@ -484,20 +484,11 @@ internal static string ADP_InvalidMixedUsageOfAccessTokenCallbackAndIntegratedSe } /// - /// Looks up a localized string similar to Cannot set the AccessToken or AccessTokenCallback property if the SspiContextProvider property has been set.. + /// Looks up a localized string similar to Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider.. /// - internal static string ADP_InvalidMixedUsageOfAccessTokenAndSspiContextProvider { + internal static string ADP_InvalidMixedUsageOfAccessTokenProperties { get { - return ResourceManager.GetString("ADP_InvalidMixedUsageOfAccessTokenAndSspiContextProvider", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Cannot set the SspiContextProvider property if the AccessToken or AccessTokenCallback property has been set.. - /// - internal static string ADP_InvalidMixedUsageOfSspiContextProviderAndAccessToken { - get { - return ResourceManager.GetString("ADP_InvalidMixedUsageOfSspiContextProviderAndAccessToken", resourceCulture); + return ResourceManager.GetString("ADP_InvalidMixedUsageOfAccessTokenProperties", resourceCulture); } } diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx index 57cbf80016..6e8a93705b 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx @@ -2082,11 +2082,8 @@ Cannot set the AccessTokenCallback property if the 'Integrated Security' connection string keyword has been set to 'true' or 'SSPI'. - - Cannot set the AccessToken or AccessTokenCallback property if the SspiContextProvider property has been set. - - - Cannot set the SspiContextProvider property if the AccessToken or AccessTokenCallback property has been set. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Cannot set the AccessTokenCallback property if 'Authentication=Active Directory Default' has been specified in the connection string. diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs index 1a48ccd293..4432ca3f6c 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs @@ -902,33 +902,41 @@ public void SspiContextProviderAndAccessTokenStateAreMutuallyExclusive() { Func> callback = (ctx, token) => Task.FromResult(new SqlAuthenticationToken("invalid", DateTimeOffset.MaxValue)); + string expectedMessage = global::Microsoft.Data.StringsHelper.GetString( + global::System.Strings.ADP_InvalidMixedUsageOfAccessTokenProperties); // Token first, then provider. using (SqlConnection conn = new("Data Source=localhost")) { conn.AccessToken = "token"; - Assert.Throws( + InvalidOperationException exception = Assert.Throws( () => conn.SspiContextProvider = new TestSspiContextProvider()); + Assert.Equal(expectedMessage, exception.Message); } using (SqlConnection conn = new("Data Source=localhost")) { conn.AccessTokenCallback = callback; - Assert.Throws( + InvalidOperationException exception = Assert.Throws( () => conn.SspiContextProvider = new TestSspiContextProvider()); + Assert.Equal(expectedMessage, exception.Message); } // Provider first, then token. using (SqlConnection conn = new("Data Source=localhost")) { conn.SspiContextProvider = new TestSspiContextProvider(); - Assert.Throws(() => conn.AccessToken = "token"); + InvalidOperationException exception = Assert.Throws( + () => conn.AccessToken = "token"); + Assert.Equal(expectedMessage, exception.Message); } using (SqlConnection conn = new("Data Source=localhost")) { conn.SspiContextProvider = new TestSspiContextProvider(); - Assert.Throws(() => conn.AccessTokenCallback = callback); + InvalidOperationException exception = Assert.Throws( + () => conn.AccessTokenCallback = callback); + Assert.Equal(expectedMessage, exception.Message); } } From baf2e6bc6fbd967fb9a5ac985a7d8e750822579a Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:42:29 -0300 Subject: [PATCH 13/51] Validate PR milestone against the target branch (#4610) --- .github/scripts/check-milestone-branch.sh | 210 ++++++++++++ .../recheck-milestones-for-release-branch.sh | 150 +++++++++ .github/scripts/tests/README.md | 12 +- .../scripts/tests/check-milestone-branch.bats | 317 ++++++++++++++++++ ...recheck-milestones-for-release-branch.bats | 228 +++++++++++++ .github/workflows/check-milestone.yml | 51 ++- .github/workflows/recheck-milestones.yml | 65 ++++ 7 files changed, 1029 insertions(+), 4 deletions(-) create mode 100755 .github/scripts/check-milestone-branch.sh create mode 100755 .github/scripts/recheck-milestones-for-release-branch.sh create mode 100644 .github/scripts/tests/check-milestone-branch.bats create mode 100644 .github/scripts/tests/recheck-milestones-for-release-branch.bats create mode 100644 .github/workflows/recheck-milestones.yml diff --git a/.github/scripts/check-milestone-branch.sh b/.github/scripts/check-milestone-branch.sh new file mode 100755 index 0000000000..3be77eec59 --- /dev/null +++ b/.github/scripts/check-milestone-branch.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# check-milestone-branch.sh +# +# Validates that a pull request's milestone is consistent with the branch the +# pull request targets. +# +# OVERVIEW +# -------- +# Milestones in this repository are named "..", optionally +# with a pre-release suffix (e.g. "7.0.3", "8.0.0-preview1"). Every milestone +# therefore maps to a candidate release branch: +# +# .. -> release/. +# +# Release branches and configured milestones determine where the work belongs: +# +# * The branch EXISTS -> that version has already forked off the default +# branch and is in servicing. Changes for it go to release/.. +# +# * The branch DOES NOT exist, and this is the earliest configured milestone +# series without a release branch -> that version is in development on the +# default branch. Changes for it go to the default branch. +# +# * Any other series -> the default branch carries exactly one development +# line, so a later series is not active yet and an earlier series is no +# longer in development. Neither may target the default branch. +# +# This rule is self-maintaining: no hard-coded version list needs updating when +# a new release branch is cut. +# +# VALIDATION MATRIX +# ----------------- +# Target branch Release branch exists? Result +# ----------------------- ----------------------- ---------------------- +# release/. n/a (it is the target) pass +# another release/* n/a fail (mismatch) +# default branch no, active line pass +# default branch no, later configured line fail (not active yet) +# default branch no, earlier configured line fail (no longer in development) +# default branch no active line configured fail (milestone missing) +# default branch yes fail (needs servicing branch) +# anything else n/a skipped (integration branch) +# +# Pull requests into long-lived integration branches (e.g. "dev/paul/foo") are +# skipped, because the milestone is enforced when that branch is merged into +# the default branch or a release branch. +# +# Milestones that don't parse as ".." are skipped with a +# notice rather than failing the build. +# +# REQUIRED ENVIRONMENT VARIABLES +# ------------------------------ +# MILESTONE_TITLE The PR's milestone title (e.g. "7.0.3"). +# BASE_REF The branch the PR targets (e.g. "main", "release/7.0"). +# DEFAULT_BRANCH The repository's default branch (e.g. "main"). +# GITHUB_REPOSITORY Owner/repo (e.g. "dotnet/SqlClient"). Set automatically by Actions. +# GH_TOKEN GitHub token for API calls (gh CLI auth). +# +# OUTPUTS +# ------- +# Emits ::notice:: on success/skip and ::error:: on failure. +# Exits 0 when the milestone and target branch agree (or the check is +# skipped), and 1 when they conflict. +# +# USAGE +# Called from the check-milestone.yml workflow. Can also be run locally: +# +# export MILESTONE_TITLE="7.0.3" +# export BASE_REF="main" +# export DEFAULT_BRANCH="main" +# export GITHUB_REPOSITORY="dotnet/SqlClient" +# bash .github/scripts/check-milestone-branch.sh +# +################################################################################# +set -euo pipefail + +# -- Runtime help ------------------------------------------------------------- +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + # Print the header comment block (between the license banner and the + # closing banner), stripping the leading '# ' prefix. + awk '/^#{2,}$/ { n++; next } n == 2 { sub(/^# ?/, ""); print }' "$0" + exit 0 +fi + +# -- Input validation --------------------------------------------------------- +: "${MILESTONE_TITLE:?MILESTONE_TITLE environment variable is required}" +: "${BASE_REF:?BASE_REF environment variable is required}" +: "${DEFAULT_BRANCH:?DEFAULT_BRANCH environment variable is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY environment variable is required}" + +# -- Derive the candidate release branch from the milestone ------------------- +# Accepts "X.Y.Z" with an optional pre-release/build suffix, e.g. "8.0.0-preview1". +if [[ "${MILESTONE_TITLE}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)([-+].*)?$ ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" +else + echo "::notice::Milestone '${MILESTONE_TITLE}' is not in 'major.minor.patch' form; skipping the target branch check." + exit 0 +fi + +RELEASE_BRANCH="release/${MAJOR}.${MINOR}" + +# -- Skip integration branches ------------------------------------------------ +# Only the default branch and release branches carry milestone semantics. +if [[ "${BASE_REF}" != "${DEFAULT_BRANCH}" && "${BASE_REF}" != release/* ]]; then + echo "::notice::PR targets integration branch '${BASE_REF}'; skipping the milestone/branch check." + exit 0 +fi + +# -- Validate a release branch target ----------------------------------------- +# The target branch itself proves which version is being serviced, so no branch +# listing is needed here. +if [[ "${BASE_REF}" == release/* ]]; then + if [[ "${BASE_REF}" != "${RELEASE_BRANCH}" ]]; then + echo "::error::Milestone '${MILESTONE_TITLE}' belongs to '${RELEASE_BRANCH}', but this PR targets '${BASE_REF}'. Retarget the PR or assign the milestone that matches '${BASE_REF}'." + exit 1 + fi + + echo "::notice::Milestone '${MILESTONE_TITLE}' matches target branch '${BASE_REF}'." + exit 0 +fi + +# -- Validate a default branch target ----------------------------------------- +# 'matching-refs' returns only refs under the given prefix, so this is a single +# cheap call regardless of how many topic branches the repository has. +if ! RELEASE_REFS=$(gh api "repos/${GITHUB_REPOSITORY}/git/matching-refs/heads/release/" \ + --jq '.[].ref' 2>&1); then + echo "::error::Unable to list release branches for '${GITHUB_REPOSITORY}': ${RELEASE_REFS}" + exit 1 +fi + +if grep -qxF "refs/heads/${RELEASE_BRANCH}" <<< "${RELEASE_REFS}"; then + echo "::error::Milestone '${MILESTONE_TITLE}' is a servicing release owned by '${RELEASE_BRANCH}', but this PR targets '${DEFAULT_BRANCH}'. Either retarget the PR to '${RELEASE_BRANCH}', or assign an in-development milestone and add the 'Hotfix ${MAJOR}.${MINOR}.${PATCH}' label so the change is cherry-picked after merge." + exit 1 +fi + +if ! MILESTONES=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/milestones?state=all&per_page=100" \ + --jq '.[].title' 2>&1); then + echo "::error::Unable to list milestones for '${GITHUB_REPOSITORY}': ${MILESTONES}" + exit 1 +fi + +ACTIVE_MAJOR="" +ACTIVE_MINOR="" +LATEST_RELEASE_MAJOR="" +LATEST_RELEASE_MINOR="" +while IFS= read -r release_ref; do + if [[ ! "${release_ref}" =~ ^refs/heads/release/([0-9]+)\.([0-9]+)$ ]]; then + continue + fi + + release_major="${BASH_REMATCH[1]}" + release_minor="${BASH_REMATCH[2]}" + if [[ -z "${LATEST_RELEASE_MAJOR}" ]] || + (( 10#${release_major} > 10#${LATEST_RELEASE_MAJOR} )) || + (( 10#${release_major} == 10#${LATEST_RELEASE_MAJOR} && 10#${release_minor} > 10#${LATEST_RELEASE_MINOR} )); then + LATEST_RELEASE_MAJOR="${release_major}" + LATEST_RELEASE_MINOR="${release_minor}" + fi +done <<< "${RELEASE_REFS}" + +while IFS= read -r milestone; do + if [[ ! "${milestone}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)([-+].*)?$ ]]; then + continue + fi + + candidate_major="${BASH_REMATCH[1]}" + candidate_minor="${BASH_REMATCH[2]}" + candidate_branch="release/${candidate_major}.${candidate_minor}" + if grep -qxF "refs/heads/${candidate_branch}" <<< "${RELEASE_REFS}"; then + continue + fi + + if [[ -n "${LATEST_RELEASE_MAJOR}" ]] && + { (( 10#${candidate_major} < 10#${LATEST_RELEASE_MAJOR} )) || + (( 10#${candidate_major} == 10#${LATEST_RELEASE_MAJOR} && 10#${candidate_minor} <= 10#${LATEST_RELEASE_MINOR} )); }; then + continue + fi + + if [[ -z "${ACTIVE_MAJOR}" ]] || + (( 10#${candidate_major} < 10#${ACTIVE_MAJOR} )) || + (( 10#${candidate_major} == 10#${ACTIVE_MAJOR} && 10#${candidate_minor} < 10#${ACTIVE_MINOR} )); then + ACTIVE_MAJOR="${candidate_major}" + ACTIVE_MINOR="${candidate_minor}" + fi +done <<< "${MILESTONES}" + +if [[ -z "${ACTIVE_MAJOR}" ]]; then + echo "::error::No configured milestone series is newer than the newest release branch, so no development line is active on '${DEFAULT_BRANCH}'. Create the milestone for the next version before targeting '${DEFAULT_BRANCH}' with '${MILESTONE_TITLE}'." + exit 1 +fi + +if (( 10#${MAJOR} != 10#${ACTIVE_MAJOR} || 10#${MINOR} != 10#${ACTIVE_MINOR} )); then + if (( 10#${MAJOR} > 10#${ACTIVE_MAJOR} || + (10#${MAJOR} == 10#${ACTIVE_MAJOR} && 10#${MINOR} > 10#${ACTIVE_MINOR}) )); then + echo "::error::Milestone '${MILESTONE_TITLE}' is for a later development line, but the ${ACTIVE_MAJOR}.${ACTIVE_MINOR} milestone series remains active on '${DEFAULT_BRANCH}' until 'release/${ACTIVE_MAJOR}.${ACTIVE_MINOR}' is cut. Assign a milestone from the active ${ACTIVE_MAJOR}.${ACTIVE_MINOR} line." + else + echo "::error::Milestone '${MILESTONE_TITLE}' is for the ${MAJOR}.${MINOR} line, which is no longer in development on '${DEFAULT_BRANCH}'; the active line is ${ACTIVE_MAJOR}.${ACTIVE_MINOR}. Assign a milestone from the active ${ACTIVE_MAJOR}.${ACTIVE_MINOR} line." + fi + exit 1 +fi + +echo "::notice::Milestone '${MILESTONE_TITLE}' is still in development (no '${RELEASE_BRANCH}' branch); targeting '${DEFAULT_BRANCH}' is correct." diff --git a/.github/scripts/recheck-milestones-for-release-branch.sh b/.github/scripts/recheck-milestones-for-release-branch.sh new file mode 100755 index 0000000000..b48ceb4ad6 --- /dev/null +++ b/.github/scripts/recheck-milestones-for-release-branch.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# recheck-milestones-for-release-branch.sh +# +# Re-runs the milestone check for open pull requests whose result can change +# when a release branch is created. +# +# OVERVIEW +# -------- +# check-milestone-branch.sh decides where a milestone belongs by asking whether +# release/. exists and which configured milestone series is the +# earliest one without a release branch. Cutting a branch therefore moves its +# X.Y.* milestones into servicing and can make the next series active. +# +# Creating a branch emits no pull request activity, so an already-open PR keeps +# whatever result it last recorded. This script closes that gap by re-running +# every semantic-version-milestoned PR targeting the default branch. That covers +# both the newly serviced series and any later series whose eligibility changes. +# +# Re-running is enough because check-milestone-branch.sh queries the live list +# of release branches. The replayed event payload still carries the correct +# milestone and base branch, since the check re-runs on every 'milestoned' and +# 'edited' activity, so the newest run always reflects the current PR state. +# +# LIMITATION +# ---------- +# A re-run replays the original run's commit, which means it executes the +# workflow definition and script as they existed then. Only the release branch +# lookup is evaluated live. +# +# So a pull request whose most recent milestone check predates the arrival of +# the target-branch rule will replay the older check, pass it, and keep its +# stale result. This is transitional: any pull request with activity after the +# rule shipped has a run that contains it. It is not detected here, because +# distinguishing a stale replay costs an extra API call per pull request and +# the window closes on its own. After the first release branch is cut following +# a change to the check itself, review the affected pull requests by hand. +# +# REQUIRED ENVIRONMENT VARIABLES +# ------------------------------ +# RELEASE_BRANCH The branch that was just created (e.g. "release/7.1"). +# DEFAULT_BRANCH The repository's default branch (e.g. "main"). +# WORKFLOW_FILE Workflow file name to re-run (e.g. "check-milestone.yml"). +# GITHUB_REPOSITORY Owner/repo (e.g. "dotnet/SqlClient"). Set automatically by Actions. +# GH_TOKEN GitHub token for API calls. Needs 'actions: write'. +# +# OUTPUTS +# ------- +# Emits ::notice:: per PR re-run and ::warning:: for any PR that could not be +# re-run. Exits 1 if at least one re-run failed, so the failure is visible in +# the Actions UI; a maintainer can then re-run those checks by hand. +# +# USAGE +# Called from the recheck-milestones.yml workflow. Can also be run locally: +# +# export RELEASE_BRANCH="release/7.1" +# export DEFAULT_BRANCH="main" +# export WORKFLOW_FILE="check-milestone.yml" +# export GITHUB_REPOSITORY="dotnet/SqlClient" +# bash .github/scripts/recheck-milestones-for-release-branch.sh +# +################################################################################# +# 'set -e' is deliberately omitted: one PR failing to re-run must not abandon +# the rest. Failures are tracked explicitly and reported at the end. +set -uo pipefail + +# -- Runtime help ------------------------------------------------------------- +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + # Print the header comment block (between the license banner and the + # closing banner), stripping the leading '# ' prefix. + awk '/^#{2,}$/ { n++; next } n == 2 { sub(/^# ?/, ""); print }' "$0" + exit 0 +fi + +# -- Input validation --------------------------------------------------------- +: "${RELEASE_BRANCH:?RELEASE_BRANCH environment variable is required}" +: "${DEFAULT_BRANCH:?DEFAULT_BRANCH environment variable is required}" +: "${WORKFLOW_FILE:?WORKFLOW_FILE environment variable is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY environment variable is required}" + +if [[ ! "${RELEASE_BRANCH}" =~ ^release/[0-9]+\.[0-9]+$ ]]; then + echo "::notice::'${RELEASE_BRANCH}' is not a 'release/.' branch; nothing to reconcile." + exit 0 +fi + +# -- Find open PRs whose result can change ------------------------------------- +if ! OPEN_PRS=$(gh pr list --repo "${GITHUB_REPOSITORY}" --base "${DEFAULT_BRANCH}" \ + --state open --limit 500 --json number,headRefOid,milestone \ + --jq '.[] | select(.milestone != null) | "\(.number) \(.headRefOid) \(.milestone.title)"' 2>&1); then + echo "::error::Unable to list open pull requests for '${GITHUB_REPOSITORY}': ${OPEN_PRS}" + exit 1 +fi + +FAILED=0 +MATCHED=0 + +while read -r NUMBER HEAD_SHA MILESTONE_TITLE; do + [[ -n "${NUMBER}" ]] || continue + + # Same milestone grammar as check-milestone-branch.sh. + [[ "${MILESTONE_TITLE}" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+].*)?$ ]] || continue + + MATCHED=$((MATCHED + 1)) + + RUNS=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?head_sha=${HEAD_SHA}&per_page=100" \ + 2>/dev/null) + + # One head commit can back PRs against several bases, so prefer the run that + # names this PR rather than just the newest run for the SHA. + RUN_ID=$(jq -r --argjson pr "${NUMBER}" \ + '([.workflow_runs[] | select(any(.pull_requests[]?; .number == $pr))] | first | .id) // empty' \ + <<< "${RUNS}" 2>/dev/null) + + # 'pull_requests' is empty for runs from forked repositories, so fall back to + # the newest run for the SHA when the association is unavailable. + if [[ -z "${RUN_ID}" ]]; then + RUN_ID=$(jq -r '.workflow_runs[0].id // empty' <<< "${RUNS}" 2>/dev/null) + fi + + if [[ -z "${RUN_ID}" ]]; then + echo "::warning::PR #${NUMBER} (milestone '${MILESTONE_TITLE}') has no milestone check run to re-run; re-check it manually." + FAILED=$((FAILED + 1)) + continue + fi + + if gh run rerun "${RUN_ID}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + echo "::notice::Re-ran the milestone check for PR #${NUMBER} (milestone '${MILESTONE_TITLE}', run ${RUN_ID})." + else + echo "::warning::Could not re-run the milestone check for PR #${NUMBER} (run ${RUN_ID}); re-check it manually." + FAILED=$((FAILED + 1)) + fi +done <<< "${OPEN_PRS}" + +if [[ "${MATCHED}" -eq 0 ]]; then + echo "::notice::No open PR targeting '${DEFAULT_BRANCH}' carries a semantic-version milestone." + exit 0 +fi + +if [[ "${FAILED}" -gt 0 ]]; then + echo "::error::${FAILED} of ${MATCHED} affected pull requests could not be re-checked automatically." + exit 1 +fi + +echo "::notice::Re-checked ${MATCHED} pull request(s) affected by '${RELEASE_BRANCH}'." diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index 9aac1c6c88..9a4ea68906 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -1,7 +1,9 @@ -# Cherry-Pick Workflow Tests +# GitHub Actions Script Tests This directory contains tests for the shell scripts used by the -[cherry-pick-hotfix](./../../../.github/workflows/cherry-pick-hotfix.yml) GitHub Actions workflow. +[cherry-pick-hotfix](./../../../.github/workflows/cherry-pick-hotfix.yml), +[check-milestone](./../../../.github/workflows/check-milestone.yml) and +[recheck-milestones](./../../../.github/workflows/recheck-milestones.yml) GitHub Actions workflows. These tests are intended to be run manually by developers when they are changing the associated scripts, and not as part of any CI runs. @@ -85,6 +87,8 @@ bats .github/scripts/tests/ ```bash bats .github/scripts/tests/extract-hotfix-versions.bats bats .github/scripts/tests/cherry-pick-to-release.bats +bats .github/scripts/tests/check-milestone-branch.bats +bats .github/scripts/tests/recheck-milestones-for-release-branch.bats ``` ### Run a specific test by name @@ -112,10 +116,12 @@ bats --formatter pretty .github/scripts/tests/ | ---- | ----- | ------ | | `extract-hotfix-versions.bats` | 18 | Label parsing, version extraction, matrix JSON output, edge cases (malformed labels, duplicates, `labeled` vs `closed` events) | | `cherry-pick-to-release.bats` | 15 | Branch derivation, already-applied detection, clean cherry-pick, conflict handling, milestone lookup, PR creation, duplicate skip logic | +| `check-milestone-branch.bats` | 26 | Milestone version parsing, state-independent active development-line selection, rejection of earlier and later series on the default branch, fail-closed handling when no series is active, release-branch derivation, default-branch vs release-branch validation, integration-branch and non-semver skips, API invocation assertions, API failure handling | +| `recheck-milestones-for-release-branch.bats` | 17 | Release-branch name parsing, matching open PRs by milestone, run lookup by head SHA and PR association, fork fallback, re-run invocation, and failure reporting | ## How the Tests Work -Both test files use the same general approach: +All test files use the same general approach: 1. **`setup()`** creates a temporary directory and populates it with mock `git` and `gh` executables — simple shell scripts that echo predetermined responses. Environment variables (`VERSION`, diff --git a/.github/scripts/tests/check-milestone-branch.bats b/.github/scripts/tests/check-milestone-branch.bats new file mode 100644 index 0000000000..813605a012 --- /dev/null +++ b/.github/scripts/tests/check-milestone-branch.bats @@ -0,0 +1,317 @@ +#!/usr/bin/env bats +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Tests for check-milestone-branch.sh +# +# Run with: bats .github/scripts/tests/check-milestone-branch.bats +# +# Dependencies: bats-core (https://github.com/bats-core/bats-core) +# +################################################################################# + +# Path to the script under test (relative to repo root). +SCRIPT=".github/scripts/check-milestone-branch.sh" + +# ── Helpers ────────────────────────────────────────────────────────────────── + +setup() { + STUB_DIR="$(mktemp -d)" + export PATH="${STUB_DIR}:${PATH}" + + # Defaults — individual tests override as needed. + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + export DEFAULT_BRANCH="main" + export GITHUB_REPOSITORY="dotnet/SqlClient" + export GH_TOKEN="fake-token" + export MOCK_MILESTONES=$'1.0.0\n2.0.1\n7.1.0\n8.0.0-preview1\n8.0.0-preview2\n8.0.0' + + mock_release_branches "release/6.1" "release/7.0" +} + +teardown() { + rm -rf "${STUB_DIR}" +} + +# Install a 'gh' mock that reports the given release branches. +mock_release_branches() { + local refs="" + local branch + for branch in "$@"; do + refs+="refs/heads/${branch}"$'\n' + done + + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +if [[ "\$*" == *"/milestones"* ]]; then + printf '%s' "\${MOCK_MILESTONES}" +else + printf '%s' '${refs}' +fi +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock that fails, simulating an API error. +mock_gh_failure() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +echo "HTTP 403: rate limit exceeded" >&2 +exit 1 +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock that lists branches but fails when milestones are queried. +mock_milestone_failure() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +if [[ "\$*" == *"/milestones"* ]]; then + echo "HTTP 403: resource not accessible by integration" >&2 + exit 1 +fi +printf '%s' 'refs/heads/release/6.1 +refs/heads/release/7.0 +' +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# ── --help flag ────────────────────────────────────────────────────────────── + +@test "prints help text with --help" { + run bash "${SCRIPT}" --help + [ "$status" -eq 0 ] + [[ "$output" == *"VALIDATION MATRIX"* ]] + [[ "$output" == *"REQUIRED ENVIRONMENT VARIABLES"* ]] +} + +@test "prints help text with -h" { + run bash "${SCRIPT}" -h + [ "$status" -eq 0 ] + [[ "$output" == *"VALIDATION MATRIX"* ]] +} + +# ── Input validation ───────────────────────────────────────────────────────── + +@test "fails when MILESTONE_TITLE is unset" { + unset MILESTONE_TITLE + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"MILESTONE_TITLE"* ]] +} + +@test "fails when BASE_REF is unset" { + unset BASE_REF + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"BASE_REF"* ]] +} + +@test "fails when DEFAULT_BRANCH is unset" { + unset DEFAULT_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"DEFAULT_BRANCH"* ]] +} + +@test "fails when GITHUB_REPOSITORY is unset" { + unset GITHUB_REPOSITORY + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"GITHUB_REPOSITORY"* ]] +} + +# ── Default branch targets ─────────────────────────────────────────────────── + +@test "passes when in-development milestone targets the default branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"::notice::"* ]] + [[ "$output" == *"still in development"* ]] +} + +@test "fails when a later milestone targets the default branch before the active release branch is cut" { + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"7.1 milestone series"* ]] + [[ "$output" == *"release/7.1"* ]] +} + +@test "closed milestone state does not change the active development line" { + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"7.1 milestone series"* ]] + [[ "$output" != *"1.0.0"* ]] + grep -qF "milestones?state=all" "${STUB_DIR}/gh.log" +} + +@test "fails when an earlier milestone series targets the default branch" { + export MILESTONE_TITLE="1.0.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"no longer in development"* ]] + [[ "$output" == *"active line is 7.1"* ]] +} + +@test "fails when no configured milestone series is active" { + mock_release_branches "release/7.0" + export MOCK_MILESTONES=$'1.0.0' + export MILESTONE_TITLE="1.0.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"no development line is active"* ]] +} + +@test "passes when a later milestone targets the default branch after the active release branch is cut" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"still in development"* ]] +} + +@test "fails when a serviced milestone targets the default branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"::error::"* ]] + [[ "$output" == *"release/7.0"* ]] + [[ "$output" == *"Hotfix 7.0.3"* ]] +} + +@test "honours a non-'main' default branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="master" + export DEFAULT_BRANCH="master" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"targeting 'master' is correct"* ]] +} + +# ── API invocation ─────────────────────────────────────────────────────────── + +@test "queries the release refs endpoint with the expected arguments" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: api repos/dotnet/SqlClient/git/matching-refs/heads/release/ --jq .[].ref" "${STUB_DIR}/gh.log" + grep -qF "GH: api --paginate repos/dotnet/SqlClient/milestones?state=all&per_page=100 --jq .[].title" "${STUB_DIR}/gh.log" +} + +@test "does not call the API when the PR targets a release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ ! -f "${STUB_DIR}/gh.log" ] +} + +# ── Release branch targets ─────────────────────────────────────────────────── + +@test "passes when the milestone matches the target release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"matches target branch 'release/7.0'"* ]] +} + +@test "fails when the milestone belongs to a different release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/6.1" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"::error::"* ]] + [[ "$output" == *"belongs to 'release/7.0'"* ]] +} + +@test "fails when an in-development milestone targets a release branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"belongs to 'release/7.1'"* ]] +} + +@test "passes when a newly cut release branch matches the milestone" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="7.1.0" + export BASE_REF="release/7.1" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"matches target branch 'release/7.1'"* ]] +} + +@test "fails on the default branch once the release branch is cut" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"release/7.1"* ]] +} + +# ── Skipped cases ──────────────────────────────────────────────────────────── + +@test "skips integration branch targets" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="dev/paul/some-feature" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"integration branch"* ]] +} + +@test "skips milestones that are not major.minor.patch" { + export MILESTONE_TITLE="vNext" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"not in 'major.minor.patch' form"* ]] +} + +@test "skips two-part milestone titles" { + export MILESTONE_TITLE="1.0 Hotfix 2" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"not in 'major.minor.patch' form"* ]] +} + +# ── API failures ───────────────────────────────────────────────────────────── + +@test "fails when the branch listing API call fails" { + mock_gh_failure + export MILESTONE_TITLE="7.0.3" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list release branches"* ]] +} + +@test "fails when the milestone listing API call fails" { + mock_milestone_failure + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list milestones"* ]] +} diff --git a/.github/scripts/tests/recheck-milestones-for-release-branch.bats b/.github/scripts/tests/recheck-milestones-for-release-branch.bats new file mode 100644 index 0000000000..0d48388f33 --- /dev/null +++ b/.github/scripts/tests/recheck-milestones-for-release-branch.bats @@ -0,0 +1,228 @@ +#!/usr/bin/env bats +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Tests for recheck-milestones-for-release-branch.sh +# +# Run with: bats .github/scripts/tests/recheck-milestones-for-release-branch.bats +# +# Dependencies: bats-core (https://github.com/bats-core/bats-core) +# +################################################################################# + +# Path to the script under test (relative to repo root). +SCRIPT=".github/scripts/recheck-milestones-for-release-branch.sh" + +# ── Helpers ────────────────────────────────────────────────────────────────── + +setup() { + STUB_DIR="$(mktemp -d)" + export PATH="${STUB_DIR}:${PATH}" + + # Defaults — individual tests override as needed. + export RELEASE_BRANCH="release/7.1" + export DEFAULT_BRANCH="main" + export WORKFLOW_FILE="check-milestone.yml" + export GITHUB_REPOSITORY="dotnet/SqlClient" + export GH_TOKEN="fake-token" + + # Open PRs as " ", one per line. + mock_gh "100 aaa111 7.1.0 +101 bbb222 8.0.0-preview1 +102 ccc333 7.1.0-preview3" + + # Default: one run per head SHA, correctly associated with its PR. + set_runs aaa111 '{"workflow_runs":[{"id":"run-aaa111","pull_requests":[{"number":100}]}]}' + set_runs bbb222 '{"workflow_runs":[{"id":"run-bbb222","pull_requests":[{"number":101}]}]}' + set_runs ccc333 '{"workflow_runs":[{"id":"run-ccc333","pull_requests":[{"number":102}]}]}' +} + +teardown() { + rm -rf "${STUB_DIR}" +} + +# Register the workflow-runs response for a given head SHA. +set_runs() { + printf '%s' "$2" > "${STUB_DIR}/runs-$1.json" +} + +# Install a 'gh' mock. $1 is the 'pr list' output; 'api' serves the JSON +# registered by set_runs, and 'run rerun' succeeds unless RERUN_FAILS is set. +mock_gh() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +case "\$1" in + pr) + printf '%s\n' '${1}' + ;; + api) + sha="\$(sed -n 's/.*head_sha=\([^&]*\).*/\1/p' <<< "\$2")" + if [[ -n "\${NO_RUN_FOUND:-}" || ! -f "${STUB_DIR}/runs-\${sha}.json" ]]; then + echo '{"workflow_runs":[]}' + else + cat "${STUB_DIR}/runs-\${sha}.json" + fi + ;; + run) + [[ -z "\${RERUN_FAILS:-}" ]] || exit 1 + ;; +esac +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock whose 'pr list' call fails. +mock_pr_list_failure() { + cat > "${STUB_DIR}/gh" <<'MOCK' +#!/usr/bin/env bash +echo "HTTP 403: rate limit exceeded" >&2 +exit 1 +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# ── --help flag ────────────────────────────────────────────────────────────── + +@test "prints help text with --help" { + run bash "${SCRIPT}" --help + [ "$status" -eq 0 ] + [[ "$output" == *"REQUIRED ENVIRONMENT VARIABLES"* ]] +} + +# ── Input validation ───────────────────────────────────────────────────────── + +@test "fails when RELEASE_BRANCH is unset" { + unset RELEASE_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"RELEASE_BRANCH"* ]] +} + +@test "fails when DEFAULT_BRANCH is unset" { + unset DEFAULT_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"DEFAULT_BRANCH"* ]] +} + +@test "fails when WORKFLOW_FILE is unset" { + unset WORKFLOW_FILE + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"WORKFLOW_FILE"* ]] +} + +@test "fails when GITHUB_REPOSITORY is unset" { + unset GITHUB_REPOSITORY + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"GITHUB_REPOSITORY"* ]] +} + +# ── Branch name parsing ────────────────────────────────────────────────────── + +@test "skips branches that are not release/." { + export RELEASE_BRANCH="dev/paul/some-feature" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"nothing to reconcile"* ]] + [ ! -f "${STUB_DIR}/gh.log" ] +} + +@test "skips a release branch with a patch component" { + export RELEASE_BRANCH="release/7.1.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"nothing to reconcile"* ]] +} + +# ── Matching and re-running ────────────────────────────────────────────────── + +@test "re-runs all semver-milestoned PRs affected by active-line transitions" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"PR #100"* ]] + [[ "$output" == *"PR #102"* ]] + [[ "$output" == *"PR #101"* ]] + [[ "$output" == *"Re-checked 3 pull request(s)"* ]] +} + +@test "queries open PRs against the default branch" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: pr list --repo dotnet/SqlClient --base main --state open" "${STUB_DIR}/gh.log" +} + +@test "looks up the run by the PR head sha and re-runs it" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "head_sha=aaa111" "${STUB_DIR}/gh.log" + grep -qF "GH: run rerun run-aaa111 --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" +} + +@test "picks the run belonging to this PR when a head sha backs several PRs" { + # The newest run for the SHA belongs to a different PR against another base. + set_runs aaa111 '{"workflow_runs":[ + {"id":"run-other","pull_requests":[{"number":999}]}, + {"id":"run-aaa111","pull_requests":[{"number":100}]} + ]}' + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: run rerun run-aaa111 --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" + ! grep -qF "run rerun run-other" "${STUB_DIR}/gh.log" +} + +@test "falls back to the newest run when the PR association is missing" { + # Runs from forked repositories carry an empty 'pull_requests' array. + set_runs aaa111 '{"workflow_runs":[ + {"id":"run-newest","pull_requests":[]}, + {"id":"run-older","pull_requests":[]} + ]}' + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"PR #100"* ]] + grep -qF "GH: run rerun run-newest --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" +} + +@test "reports when no open PR carries a matching milestone" { + export RELEASE_BRANCH="release/6.1" + mock_gh "200 ddd444 vNext" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"No open PR targeting 'main' carries a semantic-version milestone"* ]] +} + +@test "ignores PRs whose milestone is not major.minor.patch" { + mock_gh "200 ddd444 vNext" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"No open PR"* ]] +} + +# ── Failure handling ───────────────────────────────────────────────────────── + +@test "warns and fails when a PR has no run to re-run" { + export NO_RUN_FOUND=1 + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"has no milestone check run to re-run"* ]] + [[ "$output" == *"3 of 3 affected pull requests"* ]] +} + +@test "warns and fails when a re-run cannot be started" { + export RERUN_FAILS=1 + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Could not re-run the milestone check for PR #100"* ]] +} + +@test "fails when the PR listing API call fails" { + mock_pr_list_failure + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list open pull requests"* ]] +} diff --git a/.github/workflows/check-milestone.yml b/.github/workflows/check-milestone.yml index da4681fd7b..8d64ae5f13 100644 --- a/.github/workflows/check-milestone.yml +++ b/.github/workflows/check-milestone.yml @@ -1,14 +1,45 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Check Milestone +# +# Validates that every pull request has an open milestone assigned, and that +# the milestone is consistent with the branch the PR targets. +# +# Milestones map to release branches by major.minor: +# +# * "7.0.3" -> release/7.0 exists -> the PR must target release/7.0. +# * "7.1.0" -> release/7.1 does not exist and it is the earliest configured +# unbranched series -> the PR must target main. +# * "8.0.0-preview1" -> release/8.0 does not exist, but 7.1 is still the +# active unbranched series -> the PR cannot target main yet. +# +# See .github/scripts/check-milestone-branch.sh for the full rule set. +# +# Cutting release/X.Y flips the expected target for X.Y.* milestones but emits +# no pull request activity. recheck-milestones.yml reconciles the already open +# PRs that the new branch invalidates. +# +################################################################################# + name: Check Milestone on: pull_request: - types: [opened, edited, synchronize, milestoned, demilestoned] + # The 'edited' type covers base branch changes, so retargeting a PR (manually, + # or automatically when a stacked PR's parent merges) re-runs this check. + types: [opened, reopened, edited, synchronize, milestoned, demilestoned] jobs: check-milestone: name: Validate milestone runs-on: ubuntu-latest permissions: + contents: read + issues: read pull-requests: read steps: - name: Check milestone is set @@ -22,3 +53,21 @@ jobs: run: | echo "::error::Milestone '${{ github.event.pull_request.milestone.title }}' is ${{ github.event.pull_request.milestone.state }}. Please assign an open milestone." exit 1 + + - name: Checkout scripts + if: github.event.pull_request.milestone != null + uses: actions/checkout@v6 + with: + # Only the scripts directory is needed; skip full history. + sparse-checkout: .github/scripts + sparse-checkout-cone-mode: false + + - name: Check milestone matches target branch + if: github.event.pull_request.milestone != null + env: + # Pass PR data via env to avoid script injection from milestone text. + MILESTONE_TITLE: ${{ github.event.pull_request.milestone.title }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/check-milestone-branch.sh" diff --git a/.github/workflows/recheck-milestones.yml b/.github/workflows/recheck-milestones.yml new file mode 100644 index 0000000000..679b76f144 --- /dev/null +++ b/.github/workflows/recheck-milestones.yml @@ -0,0 +1,65 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Recheck Milestones +# +# Reconciles open pull requests when a release branch is cut. +# +# check-milestone.yml decides where a milestone belongs by asking whether +# release/. exists and which milestone series is active on the +# default branch. Creating a release branch can invalidate X.Y.* pull requests +# and make the next series eligible, but emits no pull request activity, so +# already open pull requests would otherwise keep their previous verdicts. +# +# This lives in its own workflow so that check-milestone.yml stays purely pull +# request scoped, and so the elevated 'actions: write' permission needed to +# re-run checks is isolated from the PR gate. +# +# A re-run replays the original run's commit, so a pull request whose last +# milestone check predates a change to the check itself replays the older +# version and keeps its stale result. See the LIMITATION section in +# .github/scripts/recheck-milestones-for-release-branch.sh. +# +# See .github/scripts/recheck-milestones-for-release-branch.sh for the details. +# +################################################################################# + +name: Recheck Milestones + +# 'create' has no branch filter, so every branch creation starts a run of this +# workflow. The guard below skips the job for anything but release/*, which is +# why this is kept out of check-milestone.yml. +on: [create] + +jobs: + recheck-open-prs: + name: Re-check open PRs after a release branch is cut + if: github.event.ref_type == 'branch' && startsWith(github.event.ref, 'release/') + runs-on: ubuntu-latest + permissions: + # 'actions: write' is needed to re-run the affected milestone checks. + actions: write + contents: read + pull-requests: read + steps: + - name: Checkout scripts + uses: actions/checkout@v6 + with: + # A 'create' run defaults to the new branch; pin the default branch so + # the script is read from a known-good copy. + ref: ${{ github.event.repository.default_branch }} + # Only the scripts directory is needed; skip full history. + sparse-checkout: .github/scripts + sparse-checkout-cone-mode: false + + - name: Re-check affected pull requests + env: + # Pass the ref via env to avoid script injection from branch names. + RELEASE_BRANCH: ${{ github.event.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + WORKFLOW_FILE: check-milestone.yml + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/recheck-milestones-for-release-branch.sh" From 5575a98b5448aa31a490149503b8831491dc0e85 Mon Sep 17 00:00:00 2001 From: Malcolm Daigle Date: Thu, 3 Sep 2026 22:20:29 -0700 Subject: [PATCH 14/51] Add perf experiment pipeline and fix v2 connection pool regressions (#4543) --- .../connection-pooling.instructions.md | 7 - eng/pipelines/perf/README.md | 138 +++++++++- eng/pipelines/perf/scripts/interleave_perf.py | 35 ++- eng/pipelines/perf/scripts/run-perf-tests.ps1 | 128 +++++++-- eng/pipelines/perf/scripts/run-perf-tests.sh | 134 ++++++++- .../perf/sqlclient-perf-experiment.yml | 239 ++++++++++++++++ .../ConnectionPool/ChannelDbConnectionPool.cs | 142 ++++++---- .../ConnectionPoolChurnRunner.cs | 83 +++++- .../ConnectionPoolContentionRunner.cs | 75 ++++- .../ConnectionPoolRampRunner.cs | 177 ++++++++++++ .../ConnectionPoolStressRunner.cs | 183 +++++++++++-- .../ConnectionPoolThreadPoolPressureRunner.cs | 256 ++++++++++++++++++ .../tests/PerformanceTests/Config/Config.cs | 2 + .../tests/PerformanceTests/Program.cs | 2 + .../tests/PerformanceTests/runnerconfig.jsonc | 18 ++ .../ChannelDbConnectionPoolTest.cs | 196 +++++++++++++- 16 files changed, 1681 insertions(+), 134 deletions(-) create mode 100644 eng/pipelines/perf/sqlclient-perf-experiment.yml create mode 100644 src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolRampRunner.cs create mode 100644 src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolThreadPoolPressureRunner.cs diff --git a/.github/instructions/connection-pooling.instructions.md b/.github/instructions/connection-pooling.instructions.md index 6464fdedd0..95019b996d 100644 --- a/.github/instructions/connection-pooling.instructions.md +++ b/.github/instructions/connection-pooling.instructions.md @@ -124,13 +124,6 @@ private readonly ChannelWriter _idleConnectionWriter; await _idleConnectionReader.WaitToReadAsync(token); ``` -### Sync Over Async Protection -```csharp -// Prevent thread pool starvation -private static SemaphoreSlim _syncOverAsyncSemaphore = - new(Math.Max(1, Environment.ProcessorCount / 2)); -``` - ## Best Practices ### Application Design diff --git a/eng/pipelines/perf/README.md b/eng/pipelines/perf/README.md index be925f5431..b7165cfc06 100644 --- a/eng/pipelines/perf/README.md +++ b/eng/pipelines/perf/README.md @@ -12,7 +12,8 @@ database. | ---- | ------- | | `sqlclient-perf-pipeline.yml` | The main (manual/nightly) pipeline. Extends `v1/Perf.Test.Job.yml@PerfTemplates`. Baseline = released NuGet package; ingests into Kusto. | | `sqlclient-perf-pr-pipeline.yml` | PR pipeline. Same template, same scripts, same options; baseline = **`main` branch source**; **no Kusto ingestion**. | -| `scripts/run-perf-tests.sh` | Linux on-VM entry point: install SDK, create DB, run benchmarks (interleaved or sequential), compare. Baseline is either a released package (`--baseline-version`) or another git ref's source (`--baseline-source-ref`). | +| `sqlclient-perf-experiment.yml` | Experiment pipeline. Same template, same scripts; both passes build the **same source** and differ only in one runner-config switch; **no Kusto ingestion**. | +| `scripts/run-perf-tests.sh` | Linux on-VM entry point: install SDK, create DB, run benchmarks (interleaved or sequential), compare. Baseline is a released package (`--baseline-version`), another git ref's source (`--baseline-source-ref`), or the same source with one runner-config switch flipped off (`--switch-under-test`). | | `scripts/run-perf-tests.ps1` | Windows equivalent (ProcessorAffinity instead of `taskset`). | | `scripts/interleave_perf.py` | Interleaved + best-of-N orchestrator: runs each unit baseline↔candidate back-to-back and confirms regressions across N passes. | | `scripts/compare_perf.py` | Compares baseline vs current BenchmarkDotNet JSON → delta (md + json). Reused by the orchestrator. | @@ -151,6 +152,122 @@ the comparison (and one removed by the PR as `removed`) instead of failing the r share the single generated runner config (`RUNNER_CONFIG` / `DATATYPES_CONFIG` env vars), so connection string and behaviour flags are identical on both sides. +## Experiment pipeline (`sqlclient-perf-experiment.yml`) + +The three perf pipelines are the same benchmarks, template and scripts pointed at three different +questions. Two of them vary the **source** under measurement; the third varies the **config**: + +| Question | Pipeline | Baseline | Current | +| --- | --- | --- | --- | +| Has this branch regressed against a released package? | `sqlclient-perf-pipeline.yml` | released NuGet package | queued branch | +| Does my PR regress the branch it merges into? | `sqlclient-perf-pr-pipeline.yml` | `main` source | queued branch | +| What does this switch cost or buy? | `sqlclient-perf-experiment.yml` | queued branch, switch **off** | queued branch, switch **on** | + +`sqlclient-perf-experiment.yml` picks one runner-config switch via the `switchUnderTest` +queue-time parameter (`UseConnectionPoolV2`, `UseOptimizedAsyncBehaviour` or +`UseManagedSniOnWindows`) and runs the baseline pass with it `false` and the current pass with it +`true`. Both passes measure the **same commit** — the branch the run is queued on — so queue it on a +PR branch to ask "what does this switch do to my change?", or on `main` to ask "what does it do to +`main`?". + +Two passes are required because these are `AppContext` switches latched process-wide (for example +`UseConnectionPoolV2` is read and cached the first time a connection pool is created), so they cannot +be toggled between benchmarks within a single process. The pipeline wires that into the existing +comparison machinery — interleaved best-of-N or sequential, via `benchmarkRunMode` — instead of two +ad hoc manual runs. + +Switch-pipeline parameters that differ from the tables above: + +| Parameter | Default | Description | +| --------- | ------- | ----------- | +| `switchUnderTest` | `UseConnectionPoolV2` | The single switch to A/B. Baseline forces it `false`, current forces it `true`. | +| `failIfSwitchSlower` | `false` | Maps onto the scripts' `--fail-on-regression` gate, but means something different here: "switch on is slower" is usually the *result* you queued the run to measure, not a defect. Enable it only when asserting the switch must not be a slowdown (e.g. before flipping its default). | +| `testTimeoutMinutes` | `180` | Same as the main pipeline, not the PR pipeline's `210`: both sides are the same source, so only **one** driver build is needed. | + +The pipeline deliberately does **not** expose `baselineVersion` / `baselineSourceRef` (the run +scripts reject combining those with `--switch-under-test`, since a simultaneous source change would +make the delta unattributable), nor the `useManagedSniOnWindows` / `useOptimizedAsyncBehaviour` / +`useConnectionPoolV2` flags. Every switch except the one under test stays at its checked-in +`runnerconfig.jsonc` value, so the measured difference is attributable to exactly one variable. + +### Why these runs are never ingested into Kusto + +This mode has its own pipeline file, rather than being a flag on the other two, specifically so that +"never ingested" is structural rather than a conditional someone can flip. Ingesting a switch +experiment would corrupt the perf database three ways: + +* **`DerivedRunId` collision.** The ID is `driver|commit|pipelineRunId`. The other two pipelines keep + their two rows distinct because the baseline row carries a *different* commit (`v7.0.2`, or the + baseline ref's sha). Here both passes are the same commit in the same pipeline run, so both rows + would derive the same ID. +* **`PerfRun.Config` is stamped once per run.** `translate_results_to_kusto.sh` builds one + `--config-override` set from the queue-time `CFG_*` values and reuses it for both the baseline and + current rows. That is correct when the config genuinely is shared, but it means the two rows could + not record the differing switch values that are the entire point of the experiment. +* **Trend pollution.** No field marks a row as an experiment — `RunType` is already + `Sequential`/`Interweaved` — so the switch-on pass would be indistinguishable from an ordinary + measurement of the branch and would distort the very trends the other two pipelines exist to + protect. + +The comparison report and the raw BenchmarkDotNet artifacts are published as usual, and the build is +tagged `Switch ` so experiments are identifiable in the ADO build list. + +### Designing benchmarks for switch experiments + +A switch experiment flips behaviour on purpose, so a benchmark that measures that behaviour will +report a regression even when the change is working. `UseConnectionPoolV2` is the motivating example: +`ChannelDbConnectionPool` opens physical connections concurrently, where `WaitHandleDbConnectionPool` +serialises growth behind a `Semaphore(1, 1)`. `ConnectionPoolStressRunner` used to call +`ClearAllPools()` in `[IterationCleanup]`, so every iteration was a cold-start burst in which the +extra parallel opens had nothing to amortise against, and the runner reported the trade-off as a loss +because that was the only thing it could measure. + +That is a benchmark defect rather than something to explain away, so the runner now pre-warms the +pool to full capacity in `[GlobalSetup]` and no longer clears it between iterations. Establishing a +connection costs milliseconds while a pooled checkout costs microseconds, so any creation left in the +measured body swamps the pool cost the runner exists to measure. Keep that separation in mind when +adding a pool benchmark: warm the pool first unless connection establishment is precisely the thing +under test. + +The pipeline has no way to mark a result as acceptable, and deliberately so: a mute is only as good +as the reasoning behind it, and that reasoning belongs in the pull request where a reviewer can +challenge it. Prefer instead to fix the benchmark, or add one that measures the intended behaviour +directly. `ConnectionPoolRampRunner` was added for exactly this reason: it keeps the cold pool but +makes every caller *hold* its connection until all of them have connected, so the pool genuinely needs +N physical connections and the only variable left is how fast it can open them. That rewards +concurrent creation instead of penalising it, and it isolates cold start now that the stress runner no +longer conflates it with checkout cost. + +The same principle applies to how a benchmark schedules its workers. A sync `Open()` that has to +wait blocks whichever thread it runs on, so a pool whose waiter wake-up needs a queued continuation +stalls when every threadpool thread is already blocked; the wake-up waits on thread injection. On +the TFMs the perf project builds (net8.0-net10.0) the runtime is told about cooperative blocking and +compensates quickly, so that stall is tens to a few hundred milliseconds, and that is the only +expectation these benchmarks validate. On net462 the `Task` wait never notifies the pool, so the +wake-up falls to starvation detection and hill climbing and is materially slower; the pool carries no +framework guards and the driver still ships net462, so that path is live but unmeasured here. +Threadpool threads are the realistic case, because sync database +calls in ASP.NET run on them, and they are the only configuration in which that stall is visible. +Benchmarks therefore keep threadpool threads as the default and add dedicated-thread variants +alongside rather than instead: + +- `ConnectionPoolContentionRunner.SteadyStateOpenQueryCloseDedicatedThreads` runs the existing + workload on dedicated threads. A regression in both variants points at the pool; a regression in + only the threadpool variant points at the waiter wake path. +- `ConnectionPoolThreadPoolPressureRunner` pins the threadpool floor via `[Params]`, below the worker + count (starved) and above it (control), so the effect is reproducible instead of depending on + hill-climbing timing. + +This failure mode is tail latency, not a shifted median, so compare distributions. Aggregating with +a per-configuration minimum hides it completely. + +Note what those benchmarks are for. Saturating the thread pool with blocked synchronous calls is an +application configuration problem, not a pool defect: an application should keep its parallelism +below the thread pool's worker count so newly queued work still runs promptly, and pre-warming the +thread pool is the application's responsibility rather than the driver's. These benchmarks exist to +characterise where that boundary sits and to catch it moving, so a delta here is a prompt to check +the boundary has not shifted rather than a bug to fix. + ## Two-pass build model The `PerformanceTests` project references Microsoft.Data.SqlClient two ways, selected by MSBuild: @@ -162,6 +279,9 @@ The `PerformanceTests` project references Microsoft.Data.SqlClient two ways, sel - **Baseline (source)**: no reference switching at all — the baseline ref's own copy of the perf project is built from `../sqlclient-perf-baseline-src`, keeping its default `ProjectReference` to that ref's driver source. Used by the PR pipeline. +- **Baseline (switch experiment)**: no second build at all — `--switch-under-test` measures one + source tree twice, so the scripts build the `current` variant once and point both passes at it, + differing only in the runner config each pass is handed. Used by the experiment pipeline. The VM's `NuGet.config` exposes only the governed feed, and CPM rejects multiple unmapped sources (`NU1507`). The baseline pass therefore restores through a **dedicated single-source config** @@ -210,7 +330,7 @@ supplies the isolated dedicated host, the tuned SQL instance, and the disjoint c | Fail loud | Preflight `SELECT 1` before any pass, **and** a post-pass guard that fails the run if a pass produced **zero** benchmark results — so an empty comparison can never be reported green. | | Warm-up | Touches the target DB in the preflight to warm the buffer pool / plan cache before the first measured benchmark. | | Allocator tuning (Linux) | Exports `MALLOC_MMAP_THRESHOLD_=128MiB` and `MALLOC_TRIM_THRESHOLD_=-1` so large-buffer benches (`LargeDataRead`, `SqlBulkCopy`) stop re-`mmap`ing per iteration. | -| Network tuning (Linux) | Best-effort `sysctl` to widen the ephemeral port range and enable `tcp_tw_reuse` for churn benches (`ConnectionPoolStress`, `ParallelAsyncConnection`). Never fails the run. | +| Network tuning (Linux) | Best-effort `sysctl` to widen the ephemeral port range and enable `tcp_tw_reuse` for churn benches (`ConnectionPoolStress`, `ConnectionPoolRamp`, `ConnectionPoolThreadPoolPressure`, `ParallelAsyncConnection`). Never fails the run. | | Diagnostics | Writes `results/diagnostics/`: SQL instance config (MAXDOP, memory, affinity, tempdb files, `@@VERSION`), host CPU topology, and per-pass CPU-clock/thermal telemetry (before/after each pass). | | Regression gate | `failOnRegression` threads `--fail-on-regression`; only a **candidate-slower** delta past the threshold fails, and in interleaved mode only after best-of-N confirmation. Default off. | | Interleaving | In `interleaved` mode the harness runs **one benchmark unit at a time, baseline then candidate back-to-back**, so both sides see the same host state (see below). | @@ -323,6 +443,18 @@ translated NDJSON as the `perf-kusto-payloads` artifact for manual/backfill inge 3. After the run, review the **run summary** (comparison, labelled `@`) and the `perf-results` artifact. The build is tagged **`Baseline `**. +### Running the experiment pipeline + +1. Open the **experiment** performance test pipeline (`sqlclient-perf-experiment.yml`) in Azure + DevOps and select **Run pipeline**. +2. Choose the branch whose behaviour you want to measure — `main` to characterise the switch on its + own, or a PR branch to characterise it against that change — and pick `switchUnderTest`. There is + no baseline selector: the baseline *is* this branch with the switch off. No Kusto configuration is + involved; these results are never ingested (see [Why these runs are never ingested into + Kusto](#why-these-runs-are-never-ingested-into-kusto)). +3. After the run, review the **run summary** (comparison, labelled `=false`) and the + `perf-results` artifact. The build is tagged **`Switch `**. + ## Troubleshooting | Symptom | Likely cause / fix | @@ -331,6 +463,8 @@ translated NDJSON as the `perf-kusto-payloads` artifact for manual/backfill inge | Baseline restore fails to find MDS | `baselineVersion` isn't a published NuGet.org version, or the VM has no outbound access to `api.nuget.org`. | | Baseline pass fails to compile: `CS1061 ... does not contain a definition for ` | A benchmark calls an MDS API newer than `baselineVersion`. Guard it with the `MDS_GE_` constants and add an older fallback — see [Benchmarks must compile against the oldest baseline](#benchmarks-must-compile-against-the-oldest-baseline). | | No comparison / summary | The baseline pass was skipped (empty `baselineVersion` / `baselineSourceRef`) or one pass produced no `*-report-full.json`. | +| `--switch-under-test is mutually exclusive with --baseline-version and --baseline-source-ref` | A switch experiment was combined with a source baseline. The experiment pipeline never does this; if you are invoking the scripts directly, clear the baseline selector — varying source and config at once makes the delta unattributable. | +| Switch experiment shows a ~0% delta everywhere | Expected for benchmarks the switch does not touch. If *every* benchmark is flat, check the run log's `Switch A/B` line actually names the switch, and that the switch is one the driver reads at startup via the runner config. | | Baseline source ref not found (PR pipeline) | `baselineSourceRef` is not a branch on `origin`, and the fallback `git clone --branch ` of `baselineRepoUrl` also failed (ref does not exist there, or the VM has no outbound access to the remote). The reason git gave is echoed into the build log and saved to `/diagnostics/git-*.log`. | | `Fetching baseline ref ... from the checkout's origin` is the last line, then the job stalls | Should no longer happen. The checkout is copied to the VM without credentials (ADO's checkout task defaults to `persistCredentials: false`), so a fetch from an authenticated `origin` used to sit on a `Username for ...` prompt forever. All network git calls now run with `GIT_TERMINAL_PROMPT=0`, no credential helper, stdin closed, and a `GIT_NET_TIMEOUT_SECS` (default 300s) hard timeout, so this fails in under a second and falls back to cloning `baselineRepoUrl`. A fetch failure here is expected and harmless on ADO. | | Ingestion step skipped | `enableKustoIngestion` is `false`, or `KustoClusterUri`, `KustoDatabase` or `KustoServiceConnection` (from `ADX Cluster Variables`) is empty (expected until the cluster is provisioned). | diff --git a/eng/pipelines/perf/scripts/interleave_perf.py b/eng/pipelines/perf/scripts/interleave_perf.py index 167002fb8f..6522adced1 100644 --- a/eng/pipelines/perf/scripts/interleave_perf.py +++ b/eng/pipelines/perf/scripts/interleave_perf.py @@ -101,17 +101,22 @@ def apply_affinity(proc, cpus): # -------------------------------------------------------------------------------------------------- # Running one unit and collecting its artifacts. # -------------------------------------------------------------------------------------------------- -def run_unit_process(exe_dir, assembly, unit, cwd, cpus, log_path): +def run_unit_process(exe_dir, assembly, unit, cwd, cpus, log_path, env_overrides=None): """Run one benchmark *unit* from the build at *exe_dir* in *cwd*. Returns the subprocess return code. Kept as a small seam so tests can substitute - a fake runner. + a fake runner. *env_overrides*, when given, is applied on top of the inherited + environment (e.g. a per-variant RUNNER_CONFIG so baseline and current can run with + different SqlClient behaviour flags, such as comparing the legacy vs new connection + pool from the SAME build). """ os.makedirs(cwd, exist_ok=True) cmd = ["dotnet", os.path.join(exe_dir, assembly)] env = dict(os.environ) env["PERF_BENCHMARK"] = unit env.pop("PERF_LIST_BENCHMARKS", None) + if env_overrides: + env.update(env_overrides) with open(log_path, "w", encoding="utf-8") as log: proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log, @@ -154,12 +159,18 @@ def collect_results(cwd, dest): class Runner: """Holds the invariant run parameters and performs interleaved unit passes.""" - def __init__(self, baseline_dir, current_dir, assembly, work_dir, cpus): + def __init__(self, baseline_dir, current_dir, assembly, work_dir, cpus, + baseline_runner_config=None, current_runner_config=None): self.baseline_dir = baseline_dir self.current_dir = current_dir self.assembly = assembly self.work_dir = work_dir self.cpus = cpus + # Optional per-variant RUNNER_CONFIG override (e.g. --switch-under-test needs baseline + # and current to run with different SqlClient behaviour flags even though they share the + # same build). None means "no override" -> both variants use the ambient RUNNER_CONFIG. + self.baseline_runner_config = baseline_runner_config + self.current_runner_config = current_runner_config def list_units(self): cmd = ["dotnet", os.path.join(self.current_dir, self.assembly)] @@ -175,7 +186,11 @@ def _run_one(self, variant, exe_dir, unit, rep, agg_dir): shutil.rmtree(cwd) os.makedirs(cwd, exist_ok=True) log_path = os.path.join(cwd, "run.log") - rc = run_unit_process(exe_dir, self.assembly, unit, cwd, self.cpus, log_path) + runner_config = (self.baseline_runner_config if variant == "baseline" + else self.current_runner_config) + env_overrides = {"RUNNER_CONFIG": runner_config} if runner_config else None + rc = run_unit_process(exe_dir, self.assembly, unit, cwd, self.cpus, log_path, + env_overrides=env_overrides) if rc != 0: _tail(log_path) raise RuntimeError(f"benchmark unit '{unit}' ({variant}, rep {rep}) failed (exit {rc}).") @@ -353,6 +368,14 @@ def main(argv=None): parser.add_argument("--reps", type=int, default=3, help="Total interleaved passes for a flagged unit (best-of-N). 1 disables confirmation.") parser.add_argument("--baseline-version", default="baseline") + parser.add_argument("--baseline-runner-config", default=None, + help="Override RUNNER_CONFIG for baseline-variant subprocesses only " + "(e.g. to force a different SqlClient behaviour flag for the " + "baseline, such as comparing the legacy vs new connection pool " + "from the same build). Omit to use the ambient RUNNER_CONFIG for " + "both variants (default behaviour).") + parser.add_argument("--current-runner-config", default=None, + help="Override RUNNER_CONFIG for current-variant subprocesses only.") parser.add_argument("--client-cpus", default=os.environ.get("PERF_CLIENT_CPUS", ""), help="CPU set to pin the benchmark client to, e.g. '16-31'.") parser.add_argument("--fail-on-regression", action="store_true", @@ -376,7 +399,9 @@ def main(argv=None): runner = Runner(os.path.abspath(args.baseline_exe_dir), os.path.abspath(args.current_exe_dir), - args.assembly, work_dir, cpus) + args.assembly, work_dir, cpus, + baseline_runner_config=args.baseline_runner_config, + current_runner_config=args.current_runner_config) units = runner.list_units() if not units: diff --git a/eng/pipelines/perf/scripts/run-perf-tests.ps1 b/eng/pipelines/perf/scripts/run-perf-tests.ps1 index c91239eb80..2b3f9bed40 100644 --- a/eng/pipelines/perf/scripts/run-perf-tests.ps1 +++ b/eng/pipelines/perf/scripts/run-perf-tests.ps1 @@ -54,7 +54,17 @@ param( [ValidateSet("", "true", "false")] [string]$UseOptimizedAsyncBehaviour = "", [ValidateSet("", "true", "false")] - [string]$UseConnectionPoolV2 = "" + [string]$UseConnectionPoolV2 = "", + # Alternative to -BaselineVersion/-BaselineSourceRef: an A/B experiment on ONE runner-config + # switch. Both passes build the SAME source; only the named switch differs (baseline=false, + # current=true), which is the only way to compare a switch whose value is latched process-wide + # (e.g. UseConnectionPoolV2 is read and cached the first time a pool is created). Mutually + # exclusive with the other two baseline selectors, and overrides the matching -Use* flag (which + # would otherwise be ambiguous: one value cannot describe two passes). ValidateSet restricts it + # to switches this script knows how to stamp, so a typo fails fast at binding time instead of + # silently writing an inert key and reporting a meaningless zero-delta comparison. + [ValidateSet("", "UseConnectionPoolV2", "UseOptimizedAsyncBehaviour", "UseManagedSniOnWindows")] + [string]$SwitchUnderTest = "" ) $ErrorActionPreference = "Stop" @@ -128,6 +138,7 @@ Write-Host " Results dir : $ResultsDir" Write-Host " Run mode : $RunMode (confirmation runs: $ConfirmationRuns)" Write-Host " Baseline ver : $(if ($BaselineVersion) { $BaselineVersion } else { '' })" Write-Host " Baseline ref : $(if ($BaselineSourceRef) { $BaselineSourceRef } else { '' })" +Write-Host " Switch A/B : $(if ($SwitchUnderTest) { "$SwitchUnderTest (baseline=false vs current=true)" } else { '' })" Write-Host " SQL_SERVER : $SqlServer" Write-Host " PERF_CLIENT_CPUS: $($env:PERF_CLIENT_CPUS)" Write-Host " PERF_SQL_CPUS : $($env:PERF_SQL_CPUS)" @@ -141,6 +152,32 @@ if (-not (Test-Path $PerfProject)) { if ((-not [string]::IsNullOrEmpty($BaselineVersion)) -and (-not [string]::IsNullOrEmpty($BaselineSourceRef))) { throw "-BaselineVersion and -BaselineSourceRef are mutually exclusive." } +if ((-not [string]::IsNullOrEmpty($SwitchUnderTest)) -and ((-not [string]::IsNullOrEmpty($BaselineVersion)) -or (-not [string]::IsNullOrEmpty($BaselineSourceRef)))) { + throw "-SwitchUnderTest is mutually exclusive with -BaselineVersion and -BaselineSourceRef: it compares the SAME source build with one switch flipped, so mixing in a source change would make the delta unattributable." +} +# UseManagedSniOnWindows only selects an implementation on Windows. PowerShell Core also runs on +# Linux, so check the host rather than assume this script implies Windows: off-Windows both passes +# use managed SNI and the experiment reports a ~0% delta that reads as "the switch is free". +# $IsWindows is undefined on Windows PowerShell 5.1, which is itself Windows-only, so null is Windows. +if ($SwitchUnderTest -eq "UseManagedSniOnWindows") { + $onWindows = if ($null -eq (Get-Variable -Name IsWindows -ErrorAction SilentlyContinue)) { $true } else { $IsWindows } + if (-not $onWindows) { + throw "-SwitchUnderTest UseManagedSniOnWindows is Windows-only; elsewhere managed SNI is always used, so baseline and current would be identical. Re-run this experiment on the Windows platform." + } +} +# -SwitchUnderTest forces its switch explicitly for each pass (baseline=false, current=true), so a +# separately-supplied -Use* flag for that SAME switch would be silently overridden; warn rather than +# let that go unnoticed. Other -Use* flags still apply normally to both passes. +$conflictingFlag = "" +$conflictingFlagValue = switch ($SwitchUnderTest) { + "UseConnectionPoolV2" { $conflictingFlag = "-UseConnectionPoolV2"; $UseConnectionPoolV2 } + "UseOptimizedAsyncBehaviour" { $conflictingFlag = "-UseOptimizedAsyncBehaviour"; $UseOptimizedAsyncBehaviour } + "UseManagedSniOnWindows" { $conflictingFlag = "-UseManagedSniOnWindows"; $UseManagedSniOnWindows } + default { "" } +} +if (-not [string]::IsNullOrEmpty($conflictingFlagValue)) { + Write-Warning "$conflictingFlag=$conflictingFlagValue is ignored when -SwitchUnderTest is $SwitchUnderTest (baseline forces false, current forces true)." +} if ([string]::IsNullOrEmpty($SqlPassword)) { throw "SQL_PASSWORD environment variable is not set (expected from the perf template)." } @@ -302,20 +339,11 @@ $env:RUNNER_CONFIG = $RunnerConfig # It needs no per-run modification, so point the env var at the checked-in file directly. $env:DATATYPES_CONFIG = Join-Path $PerfDir "datatypes.json" -$srcConfig = Join-Path $PerfDir "runnerconfig.jsonc" -$rawConfig = Get-Content $srcConfig -Raw -# Strip // line comments so ConvertFrom-Json accepts the .jsonc content. -$rawConfig = ($rawConfig -split "`n" | ForEach-Object { $_ -replace '(?m)^\s*//.*$', '' }) -join "`n" -$cfg = ConvertFrom-Json $rawConfig - # SqlClient connection-string values may be wrapped in double quotes; doubling any embedded double # quote lets a password containing ';', '=', spaces or single quotes be parsed as a single literal # value instead of corrupting the connection string. $escapedPassword = '"' + ($SqlPassword -replace '"', '""') + '"' -$cfg.ConnectionString = "Server=tcp:$SqlServer,1433;User ID=sa;Password=$escapedPassword;Initial Catalog=$DbName;TrustServerCertificate=True;Encrypt=False;" -# Apply the optional SqlClient behaviour overrides supplied by the pipeline. An empty value leaves -# the checked-in default untouched; otherwise the flag is forced to the requested boolean so the -# benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. + function Set-CfgBool { param($Config, [string]$Name, [string]$Value) if (-not [string]::IsNullOrEmpty($Value)) { @@ -324,11 +352,51 @@ function Set-CfgBool { else { $Config | Add-Member -NotePropertyName $Name -NotePropertyValue $b } } } -Set-CfgBool $cfg "UseManagedSniOnWindows" $UseManagedSniOnWindows -Set-CfgBool $cfg "UseOptimizedAsyncBehaviour" $UseOptimizedAsyncBehaviour -Set-CfgBool $cfg "UseConnectionPoolV2" $UseConnectionPoolV2 -$cfg | ConvertTo-Json -Depth 10 | Set-Content -Path $RunnerConfig -Encoding UTF8 -Write-Host "Wrote runner config to $RunnerConfig (Server=tcp:$SqlServer,1433; Initial Catalog=$DbName)" + +# Write-RunnerConfig [switchName] [switchValue] +# Writes one runner config (checked-in runnerconfig.jsonc + injected connection string + behaviour +# overrides) to . When is given, that config key is forced to +# ("true"/"false") regardless of the corresponding -Use* parameter -- used by -SwitchUnderTest, which +# needs a different value for the same switch in each pass. With no switch name the config is built +# purely from the -Use* parameters, exactly as before. +function Write-RunnerConfig { + param([string]$Dst, [string]$SwitchName = "", [string]$SwitchValue = "") + $srcConfig = Join-Path $PerfDir "runnerconfig.jsonc" + $rawConfig = Get-Content $srcConfig -Raw + # Strip // line comments so ConvertFrom-Json accepts the .jsonc content. + $rawConfig = ($rawConfig -split "`n" | ForEach-Object { $_ -replace '(?m)^\s*//.*$', '' }) -join "`n" + $cfg = ConvertFrom-Json $rawConfig + + $cfg.ConnectionString = "Server=tcp:$SqlServer,1433;User ID=sa;Password=$escapedPassword;Initial Catalog=$DbName;TrustServerCertificate=True;Encrypt=False;" + # Apply the optional SqlClient behaviour overrides supplied by the pipeline. An empty value + # leaves the checked-in default untouched; otherwise the flag is forced to the requested boolean + # so the benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. The + # switch-under-test override (when this config names one) takes precedence over the matching + # -Use* parameter, so a single checked-in template can be stamped out per pass with that one + # switch flipped and everything else identical. + $sniValue = if ($SwitchName -eq "UseManagedSniOnWindows") { $SwitchValue } else { $UseManagedSniOnWindows } + $asyncValue = if ($SwitchName -eq "UseOptimizedAsyncBehaviour") { $SwitchValue } else { $UseOptimizedAsyncBehaviour } + $poolValue = if ($SwitchName -eq "UseConnectionPoolV2") { $SwitchValue } else { $UseConnectionPoolV2 } + Set-CfgBool $cfg "UseManagedSniOnWindows" $sniValue + Set-CfgBool $cfg "UseOptimizedAsyncBehaviour" $asyncValue + Set-CfgBool $cfg "UseConnectionPoolV2" $poolValue + $cfg | ConvertTo-Json -Depth 10 | Set-Content -Path $Dst -Encoding UTF8 + Write-Host "Wrote runner config to $Dst (Server=tcp:$SqlServer,1433; Initial Catalog=$DbName)" +} + +Write-RunnerConfig -Dst $RunnerConfig + +# -SwitchUnderTest needs two DIFFERENT runner configs (baseline runs the switch off, current runs it +# on), so stamp out two more copies here alongside the shared one above. Everything else in them is +# identical, so any measured delta is attributable to the switch alone. +$BaselineRunnerConfig = "" +$CurrentRunnerConfig = "" +if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { + $BaselineRunnerConfig = Join-Path $RepoRoot "perf-runnerconfig-baseline.json" + $CurrentRunnerConfig = Join-Path $RepoRoot "perf-runnerconfig-current.json" + Write-RunnerConfig -Dst $BaselineRunnerConfig -SwitchName $SwitchUnderTest -SwitchValue "false" + Write-RunnerConfig -Dst $CurrentRunnerConfig -SwitchName $SwitchUnderTest -SwitchValue "true" +} #################################################################################################### # 4 & 5. Run the benchmarks, pinned to the reserved client CPU set. @@ -688,6 +756,11 @@ if (-not [string]::IsNullOrEmpty($BaselineVersion)) { $baselineSource = Initialize-BaselineSource -Ref $BaselineSourceRef $BaselineLabel = $baselineSource.Label $BaselineProject = $baselineSource.Project +} elseif (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { + # Switch A/B: SAME source/project for both passes ($BaselineProject/$BaselineBuildArgs are + # already the candidate's, set above), so only the runner config differs (see + # $BaselineRunnerConfig/$CurrentRunnerConfig written above: the named switch off vs on). + $BaselineLabel = "$SwitchUnderTest=false" } # Record the resolved baseline label (for a source baseline this is '@') in the results @@ -704,8 +777,17 @@ if ((-not [string]::IsNullOrEmpty($BaselineLabel)) -and ($RunMode -eq "interleav # orchestrator run one unit at a time (baseline then candidate) and confirm any flagged # regression across N passes before it counts toward the gate. #################################################################################################### - $baselineExeDir = Build-Variant "baseline" $BaselineProject $BaselineBuildArgs - $currentExeDir = Build-Variant "current" $PerfProject @() + if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { + # Switch A/B measures one build against itself with a switch flipped, so building the same + # project twice would just burn several minutes producing identical bits. Build once and + # point both variants at it; the orchestrator runs each variant in its own working directory + # (rep//), so a shared exe dir cannot cross-contaminate their artifacts. + $currentExeDir = Build-Variant "current" $PerfProject @() + $baselineExeDir = $currentExeDir + } else { + $baselineExeDir = Build-Variant "baseline" $BaselineProject $BaselineBuildArgs + $currentExeDir = Build-Variant "current" $PerfProject @() + } $interleaveArgs = @( "--baseline-exe-dir", $baselineExeDir, @@ -717,6 +799,12 @@ if ((-not [string]::IsNullOrEmpty($BaselineLabel)) -and ($RunMode -eq "interleav "--baseline-version", $BaselineLabel, "--client-cpus", "$($env:PERF_CLIENT_CPUS)" ) + # -SwitchUnderTest: baseline and current subprocesses need DIFFERENT RUNNER_CONFIG values (the + # switch off vs on), even though both are otherwise the same build/env; every other baseline + # flavour keeps sharing the single ambient RUNNER_CONFIG set above. + if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { + $interleaveArgs += @("--baseline-runner-config", $BaselineRunnerConfig, "--current-runner-config", $CurrentRunnerConfig) + } if ($FailOnRegression) { Write-Host "Regression gate ENABLED: a CONFIRMED candidate-slower regression (> $RegressionThreshold%) will fail the run." $interleaveArgs += "--fail-on-regression" @@ -726,7 +814,11 @@ if ((-not [string]::IsNullOrEmpty($BaselineLabel)) -and ($RunMode -eq "interleav } elseif (-not [string]::IsNullOrEmpty($BaselineLabel)) { # --- Legacy sequential path: full baseline pass, then full candidate pass, then compare ------- + # -SwitchUnderTest needs a different RUNNER_CONFIG per pass; every other baseline flavour + # keeps using the single ambient RUNNER_CONFIG set above (unchanged behaviour). + if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { $env:RUNNER_CONFIG = $BaselineRunnerConfig } Invoke-PerfPass "baseline" $BaselineProject $BaselineBuildArgs + if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { $env:RUNNER_CONFIG = $CurrentRunnerConfig } Invoke-PerfPass "current" $PerfProject @() Write-Host "Comparing current branch against baseline $BaselineLabel ..." diff --git a/eng/pipelines/perf/scripts/run-perf-tests.sh b/eng/pipelines/perf/scripts/run-perf-tests.sh index 8d05340342..c0b485dc9b 100755 --- a/eng/pipelines/perf/scripts/run-perf-tests.sh +++ b/eng/pipelines/perf/scripts/run-perf-tests.sh @@ -65,10 +65,22 @@ confirmationRuns="3" useManagedSniOnWindows="" useOptimizedAsyncBehaviour="" useConnectionPoolV2="" +# Alternative to --baseline-version/--baseline-source-ref: an A/B experiment on ONE runner-config +# switch. Both passes build the SAME source; only the named switch differs (baseline=false, +# current=true), which is the only way to compare a switch whose value is latched process-wide (e.g. +# UseConnectionPoolV2 is read and cached the first time a pool is created). Mutually exclusive with +# the other two baseline selectors, and overrides the matching --use-* flag (which would otherwise be +# ambiguous: one value cannot describe two passes). +switchUnderTest="" +# Runner-config switches this script is allowed to A/B. Restricted to a known list so a typo fails +# fast here instead of silently writing an inert key into the runner config and reporting a +# meaningless zero-delta comparison. +SUPPORTED_SWITCHES=("UseConnectionPoolV2" "UseOptimizedAsyncBehaviour" "UseManagedSniOnWindows") usage() { echo "Usage: $0 [--configuration ] [--framework ] [--results-subdir ]" \ - "[--baseline-version | --baseline-source-ref [--baseline-repo-url ]]" \ + "[--baseline-version | --baseline-source-ref [--baseline-repo-url ] |" \ + "--switch-under-test <${SUPPORTED_SWITCHES[*]}>]" \ "[--regression-threshold ] [--fail-on-regression]" \ "[--run-mode interleaved|sequential] [--confirmation-runs ]" \ "[--use-managed-sni-on-windows true|false] [--use-optimized-async-behaviour true|false]" \ @@ -83,6 +95,7 @@ while [[ $# -gt 0 ]]; do --baseline-version) baselineVersion="$2"; shift 2 ;; --baseline-source-ref) baselineSourceRef="$2"; shift 2 ;; --baseline-repo-url) baselineRepoUrl="$2"; shift 2 ;; + --switch-under-test) switchUnderTest="$2"; shift 2 ;; --regression-threshold) regressionThreshold="$2"; shift 2 ;; --fail-on-regression) failOnRegression="true"; shift 1 ;; --run-mode) runMode="$2"; shift 2 ;; @@ -103,12 +116,33 @@ case "${runMode}" in *) echo "ERROR: --run-mode must be 'interleaved' or 'sequential' (got '${runMode}')." >&2 usage; exit 2 ;; esac -# The two baseline selectors describe different builds of the same "baseline" pass, so requesting -# both is always a mistake; fail fast rather than silently honouring one of them. +# The three baseline selectors describe different builds/configs of the same "baseline" pass, so +# requesting more than one is always a mistake; fail fast rather than silently honouring one of them. if [[ -n "${baselineVersion}" && -n "${baselineSourceRef}" ]]; then echo "ERROR: --baseline-version and --baseline-source-ref are mutually exclusive." >&2 usage; exit 2 fi +if [[ -n "${switchUnderTest}" && ( -n "${baselineVersion}" || -n "${baselineSourceRef}" ) ]]; then + echo "ERROR: --switch-under-test is mutually exclusive with --baseline-version and --baseline-source-ref: it compares the SAME source build with one switch flipped, so mixing in a source change would make the delta unattributable." >&2 + usage; exit 2 +fi +if [[ -n "${switchUnderTest}" ]]; then + switchSupported="false" + for supported in "${SUPPORTED_SWITCHES[@]}"; do + [[ "${switchUnderTest}" == "${supported}" ]] && switchSupported="true" + done + if [[ "${switchSupported}" != "true" ]]; then + echo "ERROR: --switch-under-test must be one of: ${SUPPORTED_SWITCHES[*]} (got '${switchUnderTest}')." >&2 + usage; exit 2 + fi + # UseManagedSniOnWindows only selects an implementation on Windows. This is the bash entry + # point, so both passes here would run identical managed SNI and the experiment would spend + # hours to report a ~0% delta that looks like "the switch is free" rather than "not applicable". + if [[ "${switchUnderTest}" == "UseManagedSniOnWindows" ]]; then + echo "ERROR: --switch-under-test UseManagedSniOnWindows is Windows-only; on Linux managed SNI is always used, so baseline and current would be identical. Re-run this experiment on the Windows platform." >&2 + exit 2 + fi +fi if ! [[ "${confirmationRuns}" =~ ^[0-9]+$ ]] || [[ "${confirmationRuns}" -lt 1 ]]; then echo "ERROR: --confirmation-runs must be a positive integer (got '${confirmationRuns}')." >&2 usage; exit 2 @@ -123,6 +157,18 @@ validate_bool() { # $1 = flag name (for the message), $2 = value validate_bool use-managed-sni-on-windows "${useManagedSniOnWindows}" validate_bool use-optimized-async-behaviour "${useOptimizedAsyncBehaviour}" validate_bool use-connection-pool-v2 "${useConnectionPoolV2}" +# --switch-under-test forces its switch explicitly for each pass (baseline=false, current=true), so a +# separately-supplied --use-* flag for that SAME switch would be silently overridden; warn rather +# than let that go unnoticed. Other --use-* flags still apply normally to both passes. +case "${switchUnderTest}" in + UseConnectionPoolV2) conflictingValue="${useConnectionPoolV2}"; conflictingFlag="--use-connection-pool-v2" ;; + UseOptimizedAsyncBehaviour) conflictingValue="${useOptimizedAsyncBehaviour}"; conflictingFlag="--use-optimized-async-behaviour" ;; + UseManagedSniOnWindows) conflictingValue="${useManagedSniOnWindows}"; conflictingFlag="--use-managed-sni-on-windows" ;; + *) conflictingValue=""; conflictingFlag="" ;; +esac +if [[ -n "${conflictingValue}" ]]; then + echo "WARNING: ${conflictingFlag} is ignored when --switch-under-test is ${switchUnderTest} (baseline forces false, current forces true)." >&2 +fi #################################################################################################### # Resolve paths @@ -148,6 +194,7 @@ echo " Framework : ${framework}" echo " Results dir : ${RESULTS_DIR}" echo " Baseline ver : ${baselineVersion:-}" echo " Baseline ref : ${baselineSourceRef:-}" +echo " Switch A/B : ${switchUnderTest:-}${switchUnderTest:+ (baseline=false vs current=true)}" echo " Run mode : ${runMode} (confirmation runs: ${confirmationRuns})" echo " SQL_SERVER : ${SQL_SERVER:-}" echo " PERF_CLIENT_CPUS: ${PERF_CLIENT_CPUS:-}" @@ -295,7 +342,8 @@ export MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-134217728}" # 128 MiB export MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:--1}" # never trim # --- §2.9 Network tuning (best-effort; needs privilege, so it must never fail the run) ------------ -# Connection-churn benches (ConnectionPoolStress, ParallelAsyncConnection) exhaust ephemeral ports; +# Connection-churn benches (ConnectionPoolStress, ConnectionPoolRamp, +# ConnectionPoolThreadPoolPressure, ParallelAsyncConnection) exhaust ephemeral ports; # widen the range and allow TIME_WAIT reuse so socket setup latency stays stable. 'sudo -n' keeps # this non-interactive: on a VM without passwordless sudo it fails immediately instead of blocking # on a password prompt, then we fall back to a non-sudo sysctl (and finally give up quietly). @@ -358,10 +406,20 @@ export PERF_CFG_USE_MANAGED_SNI="${useManagedSniOnWindows}" export PERF_CFG_USE_OPTIMIZED_ASYNC="${useOptimizedAsyncBehaviour}" export PERF_CFG_USE_CONNECTION_POOL_V2="${useConnectionPoolV2}" -python3 - "$PERF_DIR/runnerconfig.jsonc" "$RUNNER_CONFIG" <<'PY' +# write_runner_config [switch_name] [switch_value] +# Writes one runner config (checked-in runnerconfig.jsonc + injected connection string + behaviour +# overrides) to . When is given, that config key is forced to +# ("true"/"false") regardless of the corresponding PERF_CFG_* value -- used by --switch-under-test, +# which needs a different value for the same switch in each pass. With no switch name the config is +# built purely from the PERF_CFG_* values, exactly as before. +write_runner_config() { + local dst="$1" + local switch_name="${2:-}" + local switch_value="${3:-}" + python3 - "$PERF_DIR/runnerconfig.jsonc" "$dst" "$switch_name" "$switch_value" <<'PY' import json, os, re, sys -src, dst = sys.argv[1], sys.argv[2] +src, dst, switch_name, switch_value = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] with open(src, "r", encoding="utf-8-sig") as fh: text = fh.read() @@ -387,13 +445,16 @@ cfg["ConnectionString"] = ( # Apply the optional SqlClient behaviour overrides supplied by the pipeline. An empty value leaves # the checked-in default untouched; otherwise the flag is forced to the requested boolean so the -# benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. +# benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. The +# switch-under-test override (when this config names one) takes precedence over the corresponding +# PERF_CFG_* value, so a single checked-in template can be stamped out per pass with that one switch +# flipped and everything else identical. for env_name, cfg_key in ( ("PERF_CFG_USE_MANAGED_SNI", "UseManagedSniOnWindows"), ("PERF_CFG_USE_OPTIMIZED_ASYNC", "UseOptimizedAsyncBehaviour"), ("PERF_CFG_USE_CONNECTION_POOL_V2", "UseConnectionPoolV2"), ): - val = os.environ.get(env_name, "") + val = switch_value if cfg_key == switch_name else os.environ.get(env_name, "") if val != "": cfg[cfg_key] = (val.lower() == "true") @@ -402,6 +463,21 @@ with open(dst, "w", encoding="utf-8") as fh: print(f"Wrote runner config to {dst} (Server=tcp:{server},1433; Initial Catalog={db})") PY +} + +write_runner_config "${RUNNER_CONFIG}" + +# --switch-under-test needs two DIFFERENT runner configs (baseline runs the switch off, current runs +# it on), so stamp out two more copies here alongside the shared one above. Everything else in them +# is identical, so any measured delta is attributable to the switch alone. +BASELINE_RUNNER_CONFIG="" +CURRENT_RUNNER_CONFIG="" +if [[ -n "${switchUnderTest}" ]]; then + BASELINE_RUNNER_CONFIG="${REPO_ROOT}/perf-runnerconfig-baseline.json" + CURRENT_RUNNER_CONFIG="${REPO_ROOT}/perf-runnerconfig-current.json" + write_runner_config "${BASELINE_RUNNER_CONFIG}" "${switchUnderTest}" "false" + write_runner_config "${CURRENT_RUNNER_CONFIG}" "${switchUnderTest}" "true" +fi #################################################################################################### # 4 & 5. Run the benchmarks, pinned to the reserved client CPU set. @@ -686,6 +762,11 @@ elif [[ -n "${baselineSourceRef}" ]]; then prepare_baseline_source "${baselineSourceRef}" baselineLabel="${BASELINE_SRC_LABEL}" baselineProject="${BASELINE_PERF_PROJECT}" +elif [[ -n "${switchUnderTest}" ]]; then + # Switch A/B: SAME source/project for both passes (baselineProject/baselineBuildArgs are already + # the candidate's, set above), so only the runner config differs (see BASELINE_RUNNER_CONFIG / + # CURRENT_RUNNER_CONFIG written above: the named switch off vs on). + baselineLabel="${switchUnderTest}=false" fi # Record the resolved baseline label (for a source baseline this is '@') in the results @@ -702,12 +783,24 @@ if [[ -n "${baselineLabel}" && "${runMode}" == "interleaved" ]]; then # orchestrator run one unit at a time (baseline then candidate) and confirm any flagged # regression across N passes before it counts toward the gate. ################################################################################################ - build_variant "baseline" "${baselineProject}" ${baselineBuildArgs[@]+"${baselineBuildArgs[@]}"} - build_variant "current" "${PERF_PROJECT}" + if [[ -n "${switchUnderTest}" ]]; then + # Switch A/B measures one build against itself with a switch flipped, so building the same + # project twice would just burn several minutes producing identical bits. Build once and + # point both variants at it; the orchestrator runs each variant in its own working directory + # (rep//), so a shared exe dir cannot cross-contaminate their artifacts. + build_variant "current" "${PERF_PROJECT}" + baselineExeDir="${REPO_ROOT}/perf-build-current" + currentExeDir="${REPO_ROOT}/perf-build-current" + else + build_variant "baseline" "${baselineProject}" ${baselineBuildArgs[@]+"${baselineBuildArgs[@]}"} + build_variant "current" "${PERF_PROJECT}" + baselineExeDir="${REPO_ROOT}/perf-build-baseline" + currentExeDir="${REPO_ROOT}/perf-build-current" + fi interleave_args=( - --baseline-exe-dir "${REPO_ROOT}/perf-build-baseline" - --current-exe-dir "${REPO_ROOT}/perf-build-current" + --baseline-exe-dir "${baselineExeDir}" + --current-exe-dir "${currentExeDir}" --assembly "PerformanceTests.dll" --results-dir "${RESULTS_DIR}" --threshold "${regressionThreshold}" @@ -715,6 +808,15 @@ if [[ -n "${baselineLabel}" && "${runMode}" == "interleaved" ]]; then --baseline-version "${baselineLabel}" --client-cpus "${PERF_CLIENT_CPUS:-}" ) + # --switch-under-test: baseline and current subprocesses need DIFFERENT RUNNER_CONFIG values (the + # switch off vs on), even though both are otherwise the same build/env; every other baseline + # flavour keeps sharing the single ambient RUNNER_CONFIG set above. + if [[ -n "${switchUnderTest}" ]]; then + interleave_args+=( + --baseline-runner-config "${BASELINE_RUNNER_CONFIG}" + --current-runner-config "${CURRENT_RUNNER_CONFIG}" + ) + fi if [[ "${failOnRegression}" == "true" ]]; then echo "Regression gate ENABLED: a CONFIRMED candidate-slower regression (> ${regressionThreshold}%) will fail the run." interleave_args+=(--fail-on-regression) @@ -724,7 +826,15 @@ if [[ -n "${baselineLabel}" && "${runMode}" == "interleaved" ]]; then elif [[ -n "${baselineLabel}" ]]; then # --- Legacy sequential path: full baseline pass, then full candidate pass, then compare ------- + # --switch-under-test needs a different RUNNER_CONFIG per pass; every other baseline flavour + # keeps using the single ambient RUNNER_CONFIG exported above (unchanged behaviour). + if [[ -n "${switchUnderTest}" ]]; then + export RUNNER_CONFIG="${BASELINE_RUNNER_CONFIG}" + fi run_pass "baseline" "${baselineProject}" ${baselineBuildArgs[@]+"${baselineBuildArgs[@]}"} + if [[ -n "${switchUnderTest}" ]]; then + export RUNNER_CONFIG="${CURRENT_RUNNER_CONFIG}" + fi run_pass "current" "${PERF_PROJECT}" echo "Comparing current branch against baseline ${baselineLabel} ..." diff --git a/eng/pipelines/perf/sqlclient-perf-experiment.yml b/eng/pipelines/perf/sqlclient-perf-experiment.yml new file mode 100644 index 0000000000..80cf11320c --- /dev/null +++ b/eng/pipelines/perf/sqlclient-perf-experiment.yml @@ -0,0 +1,239 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# SqlClient Switch Experiment Performance pipeline. +# +# Third sibling of sqlclient-perf-pipeline.yml and sqlclient-perf-pr-pipeline.yml. It runs the SAME +# benchmarks, on the SAME Perf Test Lab extends template (v1/Perf.Test.Job.yml@PerfTemplates), +# through the SAME on-VM scripts. What differs is the QUESTION it answers: +# +# * perf pipeline - "has this branch regressed against a released package?" (source varies) +# * perf-pr pipeline - "has this PR regressed the branch it merges into?" (source varies) +# * THIS pipeline - "what does this AppContext/runner switch cost or buy?" (CONFIG varies) +# +# Both passes here build the SAME source (whatever branch the run is queued on); only ONE runner +# config switch differs between them - baseline runs it OFF, current runs it ON. Two separate +# processes are required because these switches are latched process-wide (e.g. UseConnectionPoolV2 is +# read and cached the first time a connection pool is created), so they cannot be toggled between +# benchmarks inside a single run. Queue this pipeline on a PR branch to ask "what does the switch do +# to my change?", or on main to ask "what does the switch do to main?". +# +# * No Kusto - switch-experiment results are NEVER ingested into the perf database, and this is why +# the mode lives in its own pipeline rather than as a flag on the other two: +# - both PerfRun rows would share one DerivedRunId (driver|commit|pipelineRunId), +# because both passes are the same commit in the same pipeline run; +# - PerfRun.Config is stamped once per run, so the two rows could not record the +# differing switch values that are the entire point of the experiment; +# - nothing marks a row as an experiment, so the switch-ON pass would be +# indistinguishable from an ordinary measurement of the branch and would distort +# the very trends the other two pipelines exist to protect. +# Having no ADX variable group and no translate/ingest steps in this file makes that +# exclusion structural rather than a conditional someone can flip by accident. +# The comparison report and the raw BenchmarkDotNet artifacts are still published. +# +# Everything else - platform/VM provisioning, benchmark run model, noise controls - is identical to +# the other two pipelines; see eng/pipelines/perf/README.md. + +# Set the pipeline run name to the day-of-year and the daily run counter. +name: $(DayOfYear)$(Rev:rr) + +# Manual/queue-time only. A full perf run occupies a dedicated host for hours, and a switch +# experiment is an investigation someone opts into deliberately, never an automatic gate. +pr: none +trigger: none + +parameters: + + # The single runner-config switch under test. Baseline runs it false, current runs it true, so the + # reported delta is "what turning this switch ON does". Restricted to the switches the run scripts + # know how to stamp into the runner config; the scripts re-validate and fail fast on anything else. + # + # UseManagedSniOnWindows additionally requires platform=windows. ADO cannot cross-validate two + # queue-time parameters here (this pipeline extends a template, so there is no step to check it + # before the job starts), so the run scripts reject that combination as their first action. + - name: switchUnderTest + displayName: Switch under test (baseline=off vs current=on) + type: string + default: UseConnectionPoolV2 + values: + - UseConnectionPoolV2 + - UseOptimizedAsyncBehaviour + - UseManagedSniOnWindows + + # Target OS for the perf VM and the benchmark client. + - name: platform + displayName: Platform + type: string + default: linux + values: + - linux + - windows + + # The .NET runtime the benchmarks are executed against. Must be one of the target frameworks of + # the PerformanceTests project (net8.0/net9.0/net10.0). + - name: dotnetFramework + displayName: .NET Framework (TFM) + type: string + default: net9.0 + values: + - net8.0 + - net9.0 + - net10.0 + + # Maximum time (minutes) the template waits for the benchmark run on the VM before timing out. + # Matches the nightly pipeline rather than the PR pipeline: both passes here share ONE build (the + # source is identical on both sides), so there is no second driver build to pay for. + - name: testTimeoutMinutes + displayName: Test Timeout (minutes) + type: number + default: 180 + + # Percent difference (switch-on vs switch-off mean) the comparison flags. + - name: regressionThreshold + displayName: Difference Threshold (%) + type: number + default: 10 + + # Whether a confirmed "switch ON is slower than switch OFF" result should FAIL the run. + # + # NOTE: this maps onto the run scripts' --fail-on-regression gate, but it does NOT mean the same + # thing as it does in the other two pipelines. There, a regression means the branch got worse and + # failing is a quality gate. Here it is a RESULT: the switch made things slower. That is often + # exactly what you queued the run to find out, so this defaults to false and should only be enabled + # when you are asserting "this switch must not be a slowdown" (e.g. before flipping its default). + - name: failIfSwitchSlower + displayName: Fail run if the switch is slower + type: boolean + default: false + + # Benchmark run model (wiki 339 §2.2/§2.3/§2.6): + # interleaved -> run one benchmark unit at a time, switch-off and switch-on back-to-back, and + # confirm any flagged difference across N passes (best-of-N). Noise-resistant + # default, and especially valuable here because the two passes are otherwise + # identical, so any systematic drift between them is pure measurement noise. + # sequential -> legacy: run the whole switch-off suite, then the whole switch-on suite, compare. + - name: benchmarkRunMode + displayName: Benchmark run mode + type: string + default: interleaved + values: + - interleaved + - sequential + + # Best-of-N: total interleaved passes for a flagged unit before a difference is confirmed + # (1 disables confirmation). Only used when benchmarkRunMode = interleaved. + - name: confirmationRuns + displayName: Confirmation runs (best-of-N) + type: number + default: 3 + +# Fixed (non-configurable) constants for this pipeline. These are intentionally NOT parameters or +# library variables: they are invariant for the SqlClient perf pipelines. +# * buildConfiguration = Release - perf numbers are only meaningful in Release. +# * sourcesSubDir = dotnet-sqlclient - folder 'self' checks out into under the template's +# MULTI-REPO checkout ($(Build.SourcesDirectory)/); +# must match the ADO repository name. +# +# The other pipelines additionally expose UseManagedSniOnWindows / UseOptimizedAsyncBehaviour / +# UseConnectionPoolV2 as queue-time flags applied to BOTH passes. This pipeline deliberately does +# not: an experiment that varies one switch while others are also moved off their checked-in defaults +# produces a delta nobody can attribute. Every switch except the one under test is therefore left at +# its runnerconfig.jsonc default (which is what those parameters' defaults already are). +# +# NOTE: like sqlclient-perf-pr-pipeline.yml, this pipeline does NOT reference the 'ADX Cluster +# Variables' group and has no translate/ingest steps. See the header comment for why that is +# structural here rather than conditional. +variables: + + # Pre-computed testScriptArgs chunks. The Windows entry point is PowerShell and binds + # '-PascalCase' params, while the Linux one is bash and parses '--kebab-case' flags, so the + # argument STYLE differs by platform. Assembling the final args from these per-platform chunks + # keeps the single testScriptArgs below readable. No baseline-selector args appear here at all: + # --switch-under-test IS the baseline selector for this pipeline, and the run scripts reject it + # being combined with --baseline-version / --baseline-source-ref. + - ${{ if eq(parameters.platform, 'windows') }}: + - name: PerfArgsCommon + value: '-Configuration Release -Framework ${{ parameters.dotnetFramework }} -ResultsSubdir perf-results -RegressionThreshold ${{ parameters.regressionThreshold }} -RunMode ${{ parameters.benchmarkRunMode }} -ConfirmationRuns ${{ parameters.confirmationRuns }} -SwitchUnderTest ${{ parameters.switchUnderTest }}' + - ${{ else }}: + - name: PerfArgsCommon + value: '--configuration Release --framework ${{ parameters.dotnetFramework }} --results-subdir perf-results --regression-threshold ${{ parameters.regressionThreshold }} --run-mode ${{ parameters.benchmarkRunMode }} --confirmation-runs ${{ parameters.confirmationRuns }} --switch-under-test ${{ parameters.switchUnderTest }}' + + # Gate flag, only when the run is asserting the switch must not be a slowdown. + - ${{ if and(eq(parameters.platform, 'windows'), eq(parameters.failIfSwitchSlower, true)) }}: + - name: PerfArgsFail + value: '-FailOnRegression' + - ${{ elseif eq(parameters.failIfSwitchSlower, true) }}: + - name: PerfArgsFail + value: '--fail-on-regression' + - ${{ else }}: + - name: PerfArgsFail + value: '' + +# Reference the PerfTest repository that hosts the reusable extends template. Driver-team projects +# have been onboarded with the Agent Pools and Service Connections required to consume it. +resources: + repositories: + - repository: PerfTemplates + type: git + name: InternalDriverTools/PerfTest + ref: refs/heads/main + +# Consume the Perf Test Lab template. The template defines the whole job/stage structure; we only +# pass parameters into it. +extends: + template: v1/Perf.Test.Job.yml@PerfTemplates + parameters: + platform: ${{ parameters.platform }} + + # The entire driver source tree is copied to the VM so the benchmarks (which reference + # Microsoft.Data.SqlClient as a project) can be built from source against the current commit. + # Under the template's multi-repo checkout, 'self' lands in a repo-named subfolder. + testRootDir: $(Build.SourcesDirectory)/dotnet-sqlclient + + # Entry-point script (relative to testRootDir), selected by platform. Linux uses bash, Windows + # uses PowerShell, per the template's .sh/.ps1 convention. Same scripts as the other two perf + # pipelines: the switch experiment is just a different baseline selector. + ${{ if eq(parameters.platform, 'windows') }}: + testScript: eng/pipelines/perf/scripts/run-perf-tests.ps1 + ${{ else }}: + testScript: eng/pipelines/perf/scripts/run-perf-tests.sh + + # Arguments forwarded to the script. The script also reads SQL_SERVER, SQL_PASSWORD and + # PERF_CLIENT_CPUS directly from the VM session environment (injected by the template). Empty + # chunks collapse to harmless extra spaces that both PowerShell and bash arg parsing ignore. + testScriptArgs: '$(PerfArgsCommon) $(PerfArgsFail)' + + # Subfolder (relative to testRootDir on the VM) the script writes results into. The template + # copies this VM folder back to the agent, but always lands it at a fixed location: + # $(Build.ArtifactStagingDirectory)/results (and publishes it as the 'perf-results' artifact). + testResultsSubDir: perf-results + + testTimeoutMinutes: ${{ parameters.testTimeoutMinutes }} + jobName: SqlClientPerfExperiment + + # Post-test steps run on the agent after results have been copied back. The template already + # publishes the results artifact and attaches any top-level results/*.md as run summaries; this + # step additionally surfaces the BenchmarkDotNet markdown reports in the build log. There is no + # Kusto translation/ingestion here by design - see the header comment. + steps: + # Tag the build with the switch that was measured, so switch experiments are identifiable at a + # glance in the ADO build list and are never mistaken for an ordinary baseline comparison. + # Unlike the PR pipeline there is nothing to read back from the VM: the switch name is fixed at + # queue time, and both passes are the commit the run was queued on. + # NOTE: the tag intentionally has no ':' - build tags are placed in the request URL path and a + # colon trips ADO's "potentially dangerous Request.Path" filter. + - bash: | + echo "##vso[build.addbuildtag]Switch ${{ parameters.switchUnderTest }}" + displayName: 'Tag build with switch under test' + condition: succeededOrFailed() + + - task: Bash@3 + displayName: 'Show performance results' + condition: succeededOrFailed() + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)/dotnet-sqlclient/eng/pipelines/perf/scripts/show_perf_results.sh + env: + RESULTS_DIR: $(Build.ArtifactStagingDirectory)/results diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs index a547ce0fd4..9017f3f920 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs @@ -62,12 +62,6 @@ namespace Microsoft.Data.SqlClient.ConnectionPool internal sealed class ChannelDbConnectionPool : IDbConnectionPool, IDisposable { #region Fields - // Limits synchronous operations which depend on async operations on managed - // threads from blocking on all available threads, which would stop async tasks - // from being scheduled and cause deadlocks. Use ProcessorCount/2 as a balance - // between sync and async tasks. - private static SemaphoreSlim _syncOverAsyncSemaphore = new(Math.Max(1, Environment.ProcessorCount / 2)); - /// /// Tracks the number of instances of this class. Used to generate unique IDs for each instance. /// @@ -939,11 +933,23 @@ public bool TryGetConnection( if (taskCompletionSource is null) { // We're on the caller's thread, so the ambient transaction is directly observable. + Transaction? currentTransaction = ADP.GetCurrentTransaction(); + + // Fast path: when the pool can satisfy the request immediately, do it here rather + // than entering GetInternalConnection, which allocates a Task + // and a timer-backed CancellationTokenSource before it knows whether it will ever + // need to wait. See TryGetPooledConnectionInline. + connection = TryGetPooledConnectionInline(owningObject, currentTransaction); + if (connection is not null) + { + return true; + } + var task = GetInternalConnection( owningObject, async: false, timeout, - ADP.GetCurrentTransaction()); + currentTransaction); // When running synchronously, we are guaranteed that the task is already completed. // We don't need to guard the managed threadpool at this spot because we pass the async flag as false @@ -975,29 +981,21 @@ public bool TryGetConnection( // // The ambient transaction is captured by the caller, on the caller's thread, and handed // to us in the TaskCompletionSource's AsyncState (see SqlConnection.InternalOpenAsync). - // - // We must not read Transaction.Current inside the Task.Run below. A - // TransactionScope created with TransactionScopeAsyncFlowOption.Enabled stores the - // transaction in an AsyncLocal, which does flow onto the pool's worker thread, so that - // would appear to work. But Enabled is not the default: a plain TransactionScope keeps - // the transaction in thread-static storage, which does not flow, and reading - // Transaction.Current on the worker would silently fail to enlist. The WaitHandle pool - // enlists correctly in that case, so this is also a compatibility requirement. - // AsyncState is correct under both options. - // - // This does not make the suppressed-flow pattern work -- the caller's own scope is - // still broken past the first await -- but it keeps the connection in the transaction - // the caller intended rather than silently running outside it. - // - // Note that we deliberately do not assign Transaction.Current on the thread pool - // thread either. That assignment writes to thread-static storage which is *not* unwound - // when the ExecutionContext is restored, so it would outlive this open and be observed - // by unrelated work later scheduled onto the same thread pool thread -- including the - // login-time auto-enlistment that non-pooled connections perform against - // Transaction.Current. The WaitHandle pool can get away with assigning it because it - // processes pending opens on a dedicated non-thread-pool thread. + // Do not read Transaction.Current in the Task.Run below: a plain TransactionScope keeps + // the transaction in thread-static storage, which does not flow to the worker, so the + // enlistment would silently be skipped. Do not assign Transaction.Current there either; + // that storage is not unwound afterwards and would leak into later work on that thread. Transaction? ambientTransaction = taskCompletionSource.Task.AsyncState as Transaction; + // Fast path: return true rather than completing the TaskCompletionSource, so + // InternalOpenAsync takes its sync branch and skips a thread pool dispatch. + DbConnectionInternal? pooled = TryGetPooledConnectionInline(owningObject, ambientTransaction); + if (pooled is not null) + { + connection = pooled; + return true; + } + Task.Run(async () => { if (taskCompletionSource.Task.IsCompleted) @@ -1431,6 +1429,66 @@ private void RemoveConnection(DbConnectionInternal connection) return null; } + /// + /// Attempts to satisfy a connection request from connections the pool already holds, + /// without blocking, waiting, or opening a physical connection. + /// + /// + /// The fast path shared by the sync and async entry points of + /// : it tries the transacted store, then the idle channel, and + /// returns null if neither can satisfy the request. It is kept separate from + /// because that method allocates a + /// and a timer-backed even + /// when it completes synchronously. It never calls + /// , which would block on network I/O. + /// + /// The DbConnection that will own this internal connection. + /// The ambient transaction captured on the caller's thread, + /// or null when the caller is not inside a transaction. + /// An activated connection ready to be handed to the caller, or null when the pool + /// cannot satisfy the request without waiting or opening. + /// + /// Propagates any exception from activating or enlisting the connection. The connection is + /// returned to the pool before the exception escapes (see ). + /// + private DbConnectionInternal? TryGetPooledConnectionInline( + DbConnection owningConnection, + Transaction? ambientTransaction) + { + // When automatic enlistment is disabled the connection must never be bound to the + // ambient transaction, so we neither consult the transacted store nor hand the + // transaction to activation. Mirrors GetInternalConnection. + Transaction? transaction = HasTransactionAffinity ? ambientTransaction : null; + + DbConnectionInternal? connection = null; + + // A connection already enlisted in our transaction is always preferred, since reusing + // it avoids promoting the transaction to a distributed one. + if (transaction is not null) + { + connection = GetFromTransactedPool(transaction); + } + + // GetIdleConnection only returns connections that passed IsLiveConnection, and + // GetFromTransactedPool has already probed liveness, so no further validation is + // needed here. GetInternalConnection re-checks after its channel wait because that + // wait can hand back a connection that bypassed both filters. + connection ??= GetIdleConnection(); + + if (connection is null) + { + return null; + } + + // Counted before activation for the same reason as GetInternalConnection: if + // PrepareConnection fails it returns the connection to the pool, which emits the + // matching soft disconnect. Counting after would leave that disconnect unpaired and + // drive the active-soft-connects gauge negative. + Metrics.SoftConnectRequest(); + PrepareConnection(owningConnection, connection, transaction); + return connection; + } + /// /// Gets an internal connection from the pool, either by retrieving an idle connection or opening a new one. /// @@ -1681,30 +1739,10 @@ private void SweepEmancipatedConnections() /// The connection read from the channel. private DbConnectionInternal? ReadChannelSyncOverAsync(CancellationToken cancellationToken) { - // If there are no connections in the channel, then ReadAsync will block until one is available. - // Channels doesn't offer a sync API, so running ReadAsync synchronously on this thread may spawn - // additional new async work items in the managed thread pool if there are no items available in the - // channel. We need to ensure that we don't block all available managed threads with these child - // tasks or we could deadlock. Prefer to block the current user-owned thread, and limit throughput - // to the managed threadpool. - - _syncOverAsyncSemaphore.Wait(cancellationToken); - try - { - ConfiguredValueTaskAwaitable.ConfiguredValueTaskAwaiter awaiter = - _idleChannel.ReadAsync(cancellationToken).ConfigureAwait(false).GetAwaiter(); - using ManualResetEventSlim mres = new ManualResetEventSlim(false, 0); - - // Cancellation happens through the ReadAsync call, which will complete the task. - // Even a failed task will complete and set the ManualResetEventSlim. - awaiter.UnsafeOnCompleted(() => mres.Set()); - mres.Wait(CancellationToken.None); - return awaiter.GetResult(); - } - finally - { - _syncOverAsyncSemaphore.Release(); - } + // Channel has no blocking read. Block on the Task rather than an opaque primitive: the + // idle channel is created without AllowSynchronousContinuations, so the completing + // continuation is queued, and Task blocking lets the thread pool inject a worker to run it. + return _idleChannel.ReadAsync(cancellationToken).AsTask().GetAwaiter().GetResult(); } /// diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolChurnRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolChurnRunner.cs index ca1930e451..d2b94cea14 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolChurnRunner.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolChurnRunner.cs @@ -20,6 +20,22 @@ namespace Microsoft.Data.SqlClient.PerformanceTests /// concern for the new ChannelDbConnectionPool, which aims to avoid extra allocations /// on the hot path (issue #3356). /// + /// This overlaps in shape — + /// the inner loop is identical — but not in purpose, and the two are not + /// interchangeable. Being single-threaded, this runner has no scheduling or wake-up + /// component, which makes it far more sensitive: its sync and async variants have + /// agreed to within 0.3 percentage points, whereas the concurrent runner's own + /// duplicate-workload parameter pairs have disagreed by around 20. Use this one to + /// decide whether per-checkout cost or allocation moved, and the concurrent runner to + /// decide whether hand-off between threads moved. A regression there with a flat result + /// here points at scheduling rather than at the checkout path. + /// + /// Adding a Parallelism=1 case to that runner would not replace this one. Its + /// [Params] are class-wide, so a single-threaded row would also be generated for + /// benchmarks it makes meaningless (MixedSyncAsyncContention would have no sync + /// worker left to mix in), and this runner's higher operation count and separate + /// iteration settings are what keep the signal clean. + /// /// The pool implementation (legacy vs V2) is a process-level choice — see the remarks /// on . Run twice (UseConnectionPoolV2 false /// then true) to compare. @@ -36,6 +52,27 @@ public class ConnectionPoolChurnRunner : BaseRunner [Params(1000)] public int OpsPerInvocation { get; set; } + /// + /// How many idle connections the pool holds while the single caller churns against it. + /// + /// + /// This is not a redundant axis, because the two pools order idle connections + /// differently: the legacy pool pops from a ConcurrentStack (LIFO) while the V2 + /// pool reads from an unbounded Channel (FIFO). At depth 1 that difference is + /// invisible, since both hand back the only connection there is. At a realistic depth + /// the legacy pool keeps returning the connection just released — one hot object — while + /// the V2 pool cycles through every idle connection in turn, touching all of their + /// buffers and parser state. Depth is therefore the axis that exposes reuse locality, + /// and measuring only depth 1 would hide it entirely. + /// + /// The middle depth is what makes the axis diagnostic rather than merely directional. + /// Depths 1 and 100 alone show only that reuse locality degrades somewhere in between; + /// they cannot distinguish a threshold (cost appears once the pool exceeds some working + /// set) from a gradient (cost grows smoothly with depth). Those imply different fixes. + /// + [Params(1, 10, 100)] + public int PoolDepth { get; set; } + private string _connectionString; [GlobalSetup] @@ -51,16 +88,48 @@ public void Setup() { Pooling = true, MaxPoolSize = 100, - // Pre-warm a single connection so the very first checkout is served from - // the pool rather than establishing a physical connection. - MinPoolSize = 1 + // Pin the floor at the requested depth so pruning cannot shrink the pool back + // down mid-run and change what the benchmark is measuring. + MinPoolSize = PoolDepth, + // Matches ConnectionPoolStressRunner. At the larger PoolDepth, setup establishes + // a hundred physical connections back to back, and the default 15s is tight for + // that against a loaded remote server. + ConnectTimeout = 60 }; _connectionString = builder.ConnectionString; - // Force the pool to exist and hold at least one idle connection. - using var warm = new SqlConnection(_connectionString); - warm.Open(); - warm.Close(); + PrewarmPool(); + } + + /// + /// Fills the pool with idle connections before measurement, so + /// every measured open is a pure checkout. + /// + /// + /// All connections are opened before any is released. Releasing as we go would let the + /// pool hand the same idle connection straight back and create only one, which would + /// silently collapse every depth to 1. MinPoolSize alone is not enough either: + /// both pools backfill it on a background task, so the first measured iteration would + /// otherwise still be racing that warm-up. + /// + private void PrewarmPool() + { + var warm = new SqlConnection[PoolDepth]; + try + { + for (int i = 0; i < warm.Length; i++) + { + warm[i] = new SqlConnection(_connectionString); + warm[i].Open(); + } + } + finally + { + foreach (var conn in warm) + { + conn?.Dispose(); + } + } } [GlobalCleanup] diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolContentionRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolContentionRunner.cs index 899d25f64c..2dbd6f94c8 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolContentionRunner.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolContentionRunner.cs @@ -30,6 +30,14 @@ namespace Microsoft.Data.SqlClient.PerformanceTests /// remarks on . Run twice (UseConnectionPoolV2 /// false then true) to compare. /// + /// Each sync workload is measured twice, once on threadpool threads + /// () and once on dedicated threads + /// (). Threadpool threads are the + /// realistic case, since sync database calls in ASP.NET run on them, and they are the + /// only configuration that can expose a waiter wake path which depends on the + /// threadpool having a free thread. Dedicated threads isolate the pool's own cost. The + /// pair separates a pool regression from a scheduling one. + /// /// Related issues: #601, #979, #3356 /// public class ConnectionPoolContentionRunner : BaseRunner @@ -41,13 +49,24 @@ public class ConnectionPoolContentionRunner : BaseRunner public int Parallelism { get; set; } /// - /// Maximum pool size, exercised across three regimes relative to - /// : larger than the worker count (idle spare - /// connections, no contention), equal to it (fully subscribed, no contention), and - /// smaller than it (pool exhaustion forces workers to wait for a connection to be - /// returned — back-pressure). + /// Maximum pool size, exercised across the full demand/capacity ladder relative to + /// : larger than the worker count (idle spare connections, no + /// contention), equal to it (fully subscribed, no contention), and progressively smaller + /// than it (pool exhaustion forces workers to wait for a connection to be returned — + /// back-pressure). /// - [Params(100, 50, 10)] + /// + /// At 50 these give demand/capacity ratios of 0.25, 0.5, 1, 2 + /// and 5. The intermediate over-subscribed step (ratio 2) is deliberate: back-pressure + /// regressions show up only once demand exceeds capacity, and without a point between + /// "fully subscribed" and "five times over" there is no way to tell how quickly the cost + /// grows once the pool starts running dry. + /// + /// 200 is included because it is the most commonly configured explicit Max Pool Size in + /// production, so the least-contended row corresponds to a real deployment rather than + /// only to the driver default. + /// + [Params(200, 100, 50, 25, 10)] public int MaxPoolSize { get; set; } /// @@ -114,6 +133,50 @@ public Task SteadyStateOpenQueryClose() return Task.WhenAll(tasks); } + /// + /// Same workload as , but driven by dedicated + /// threads instead of threadpool threads. + /// + /// Read the two together. A sync Open() that has to wait blocks whichever + /// thread it is running on. On threadpool threads that competes with the threadpool + /// itself, because a pool implementation whose waiter wake-up depends on a queued + /// continuation cannot make progress while every thread is blocked in a wait: the + /// wake-up is stuck behind thread injection. On the TFMs this project builds the + /// runtime notifies the pool of cooperative blocking and compensates quickly, so the + /// stall is tens to a few hundred milliseconds; that is the expectation measured here. + /// On net462 the Task wait never notifies the pool, so the wake-up waits on starvation + /// detection and hill climbing instead. That path is live, since the pool carries no + /// framework guards and the driver still ships net462, but this suite cannot measure it. + /// Dedicated threads remove that coupling, so this variant measures the pool's + /// intrinsic checkout/return cost with the scheduler taken out of the picture. + /// + /// A regression in both points at the pool itself. A regression only in the + /// threadpool variant points at the waiter wake path and shows up as tail latency + /// rather than a shifted median, so compare the distribution and not just the mean. + /// + [Benchmark] + public Task SteadyStateOpenQueryCloseDedicatedThreads() + { + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Factory.StartNew(() => + { + for (int op = 0; op < OpsPerWorker; op++) + { + using var conn = new SqlConnection(_connectionString); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT 1"; + _ = cmd.ExecuteScalar(); + // Dispose returns the connection to the pool. + } + }, TaskCreationOptions.LongRunning); + } + + return Task.WhenAll(tasks); + } + [Benchmark] public Task SteadyStateOpenQueryCloseAsync() { diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolRampRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolRampRunner.cs new file mode 100644 index 0000000000..344884fbc3 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolRampRunner.cs @@ -0,0 +1,177 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Data.SqlClient.PerformanceTests +{ + /// + /// Measures how quickly a cold pool can ramp up to N physical connections when N + /// callers arrive simultaneously and all of them need a connection at the same time. + /// + /// This is the workload the ChannelDbConnectionPool (V2) was designed for. The legacy + /// WaitHandleDbConnectionPool guards creation with a Semaphore(1, 1), so a cold + /// burst of N callers establishes physical connections one at a time: total latency is + /// roughly N x connect latency. V2 has no such gate, so the opens overlap and total + /// latency approaches a single connect. + /// + /// Contrast with , which also + /// starts from a cold pool but releases each connection immediately. Because nothing is + /// held, one physical connection can satisfy every caller in turn, so that benchmark + /// rewards a pool that grows as slowly as possible and penalizes concurrent creation. + /// Holding each connection until every caller has one removes that artifact: the pool + /// genuinely needs N connections, and the only variable left is how fast it can open + /// them. + /// + /// is always larger than so no + /// caller ever waits for a connection to be returned. Back-pressure on a saturated pool + /// is covered separately by . + /// + /// The pool implementation (legacy vs V2) is a process-level choice - see the remarks on + /// . Run twice (UseConnectionPoolV2 false then + /// true) to compare. + /// + /// Related issue: #3356 + /// + public class ConnectionPoolRampRunner : BaseRunner + { + /// + /// Number of callers that arrive simultaneously against a cold pool. Each one holds + /// its connection until all of them have connected, so the pool must open exactly + /// this many physical connections. + /// + [Params(10, 25, 50, 100)] + public int Parallelism { get; set; } + + /// + /// Max pool size. Deliberately larger than every value so + /// the ramp is never bounded by pool capacity. + /// + [Params(200)] + public int MaxPoolSize { get; set; } + + private string _connectionString; + + /// + /// Upper bound on the rendezvous wait. The rendezvous is released on the failure path + /// too, so this should never be reached; it exists so that a bug in the barrier fails + /// the run quickly instead of hanging the perf pipeline indefinitely. + /// + private static readonly TimeSpan s_rampTimeout = TimeSpan.FromMinutes(2); + + [GlobalSetup] + public void Setup() + { + Console.WriteLine( + "[ConnectionPoolRampRunner] Pool implementation: " + + (s_config.UseConnectionPoolV2 + ? "ChannelDbConnectionPool (V2)" + : "WaitHandleDbConnectionPool (legacy)")); + + var builder = new SqlConnectionStringBuilder(s_config.ConnectionString) + { + Pooling = true, + MaxPoolSize = MaxPoolSize, + // No pre-warming: every iteration must establish its own connections. + MinPoolSize = 0, + ConnectTimeout = 60 + }; + _connectionString = builder.ConnectionString; + } + + [IterationSetup] + public void IterationSetup() + { + // Start every iteration from a cold pool so the measurement is the ramp itself. + SqlConnection.ClearAllPools(); + } + + [GlobalCleanup] + public void Cleanup() => SqlConnection.ClearAllPools(); + + /// + /// Async cold-start ramp. All callers open concurrently and hold until the last one + /// has connected. + /// + [Benchmark] + public async Task ColdStartRampAsync() + { + using var allConnected = new CountdownEvent(Parallelism); + var release = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Run(async () => + { + using var conn = new SqlConnection(_connectionString); + try + { + await conn.OpenAsync(); + } + finally + { + // Signal from a finally so a caller that fails to open still counts as + // arrived. Otherwise the countdown never reaches zero, the release is + // never set, and every other caller awaits forever. + if (allConnected.Signal()) + { + release.TrySetResult(true); + } + } + + // Hold the connection until every caller has one, forcing the pool to + // grow to Parallelism physical connections. + await release.Task.WaitAsync(s_rampTimeout); + // Dispose returns the connection to the pool. + }); + } + + await Task.WhenAll(tasks); + } + + /// + /// Sync cold-start ramp. Uses dedicated threads rather than thread pool threads so + /// the measurement reflects pool ramp latency rather than thread pool injection + /// delay, which would otherwise dominate once the callers block. + /// + [Benchmark] + public void ColdStartRamp() + { + using var allConnected = new CountdownEvent(Parallelism); + + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Factory.StartNew(() => + { + using var conn = new SqlConnection(_connectionString); + try + { + conn.Open(); + } + finally + { + // See ColdStartRampAsync: signalling from a finally keeps a failed + // open from stranding every other caller in Wait(). + allConnected.Signal(); + } + + if (!allConnected.Wait(s_rampTimeout)) + { + throw new TimeoutException( + $"Cold-start ramp did not reach {Parallelism} connections within {s_rampTimeout}."); + } + // Dispose returns the connection to the pool. + }, TaskCreationOptions.LongRunning); + } + + Task.WaitAll(tasks); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolStressRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolStressRunner.cs index 9dad6f41d0..e65e08d22f 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolStressRunner.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolStressRunner.cs @@ -10,12 +10,20 @@ namespace Microsoft.Data.SqlClient.PerformanceTests { /// /// Stress-tests the connection pool with randomized parallel access patterns: - /// - Massive concurrent open/close churn + /// - Massive concurrent open/close churn, both sync and async /// - Randomized hold durations simulating real workloads /// - Mixed sync/async callers competing for pooled connections /// - Connection reuse with interleaved queries /// - Pool exhaustion and recovery under pressure /// + /// The pool is pre-warmed to full capacity in and is deliberately + /// not cleared between iterations, so these benchmarks measure steady-state + /// checkout and return rather than physical connection establishment. Establishing + /// connections costs orders of magnitude more than a pooled checkout (milliseconds versus + /// microseconds), so leaving any creation in the measured body swamps the pool cost this + /// class exists to measure. Cold-start behaviour is covered separately and deliberately by + /// . + /// /// Related issues: #601, #979, #3356 /// public class ConnectionPoolStressRunner : BaseRunner @@ -23,24 +31,59 @@ public class ConnectionPoolStressRunner : BaseRunner private string _connectionString; private string _tableName; + /// + /// Total checkouts performed by and + /// , held constant across every + /// and combination so a spread between cells reflects the pool, + /// not a different amount of work. Divides evenly by both Parallelism values. + /// + private const int RapidFireCheckouts = 1000; + /// /// Number of concurrent tasks hammering the pool. /// - [Params(10, 20, 25)] + /// + /// Two values spanning 5x rather than a tighter cluster. Every benchmark here runs at + /// every , so this axis is multiplied across seven benchmarks + /// and intermediate values buy far less than they cost — adjacent values in a narrow + /// range mostly re-measure the same regime. + /// + [Params(10, 50)] public int Parallelism { get; set; } /// /// Max pool size — controls how many physical connections the pool can hold. - /// When Parallelism exceeds this, tasks must wait for a free connection. /// + /// + /// The pool is pre-warmed to this many connections and pinned there by an equal + /// Min Pool Size, so this is the number of connections actually resident for the + /// whole run rather than just a ceiling. The limit itself is only reached by + /// , which deliberately oversubscribes it, and by the + /// fully-subscribed corner where equals this value. Everywhere + /// else the pool never saturates, so this parameter mostly varies how many idle + /// connections the checkout path is choosing among; treat a spread between two + /// MaxPoolSize values at the same Parallelism as a noise estimate rather than a real + /// effect. The fully-subscribed corner is the exception — there the spread is a real + /// effect, because one side has idle spares to choose from and the other has none. + /// + /// This reading holds for every benchmark here except + /// , whose task count is MaxPoolSize + + /// Parallelism: at Parallelism 10 the two cells run 60 and 110 tasks, so their spread + /// is mostly the extra work. That coupling is intrinsic, since exhausting the pool means + /// first saturating it. Elsewhere, do not make a benchmark body's operation count a + /// function of MaxPoolSize, or the spread stops being a noise estimate there too. + /// [Params(50, 100)] public int MaxPoolSize { get; set; } [GlobalSetup] public void Setup() { + // Pin Min Pool Size to Max Pool Size so the pool holds full capacity for the whole + // run: pruning cannot shrink it back down, and no benchmark body has to establish a + // physical connection. _connectionString = s_config.ConnectionString + - $";Pooling=True;Max Pool Size={MaxPoolSize};Min Pool Size=5;Connect Timeout=60"; + $";Pooling=True;Max Pool Size={MaxPoolSize};Min Pool Size={MaxPoolSize};Connect Timeout=60"; // Create a small table for query workloads. // Hash the machine name instead of using it verbatim: hostnames can be long enough @@ -48,22 +91,55 @@ public void Setup() // than using Math.Abs, which throws OverflowException when the hash is int.MinValue. string machineHash = ((uint)Environment.MachineName.GetHashCode()).ToString("x8"); _tableName = $"[perf_PoolStress_{machineHash}_{Guid.NewGuid():N}]"; - using var conn = new SqlConnection(_connectionString); - conn.Open(); - using var cmd = new SqlCommand( - $"CREATE TABLE {_tableName} (Id INT IDENTITY PRIMARY KEY, Val INT)", conn); - cmd.ExecuteNonQuery(); - // Seed a few rows so SELECT queries return data - using var insert = new SqlCommand( - $"INSERT INTO {_tableName} (Val) VALUES (1),(2),(3),(4),(5)", conn); - insert.ExecuteNonQuery(); + // Scoped so this connection is back in the pool before PrewarmPool runs: that method + // holds MaxPoolSize connections at once, so an extra live one would hit the cap and + // block until Connect Timeout. + { + using var conn = new SqlConnection(_connectionString); + conn.Open(); + using var cmd = new SqlCommand( + $"CREATE TABLE {_tableName} (Id INT IDENTITY PRIMARY KEY, Val INT)", conn); + cmd.ExecuteNonQuery(); + + // Seed a few rows so SELECT queries return data + using var insert = new SqlCommand( + $"INSERT INTO {_tableName} (Val) VALUES (1),(2),(3),(4),(5)", conn); + insert.ExecuteNonQuery(); + } + + PrewarmPool(); } - [IterationCleanup] - public void IterationCleanup() + /// + /// Fills the pool to live connections before any measurement + /// starts, so benchmark bodies only ever exercise checkout and return. + /// + /// + /// Every connection is opened before any is released. Releasing as we go would let the + /// pool hand the same idle connection back repeatedly and create only one, which is the + /// whole failure this is meant to avoid. Min Pool Size alone is not enough either: + /// it is backfilled lazily, so the first measured iteration would still pay for creation. + /// + private void PrewarmPool() { - SqlConnection.ClearAllPools(); + var warm = new SqlConnection[MaxPoolSize]; + try + { + for (int i = 0; i < warm.Length; i++) + { + warm[i] = new SqlConnection(_connectionString); + warm[i].Open(); + } + } + finally + { + // Returns them all to the pool; Min Pool Size keeps them resident from here on. + foreach (var conn in warm) + { + conn?.Dispose(); + } + } } [GlobalCleanup] @@ -79,14 +155,31 @@ public void Cleanup() /// /// Pure open/close churn — every task opens a pooled connection, immediately closes it, - /// and repeats. Measures raw pool checkout/return throughput under contention. - /// The per-task loop count scales with MaxPoolSize so total checkouts stay proportional - /// to pool capacity regardless of how the [Params] values change. + /// and repeats. With the pool pre-warmed to full capacity, every open is a pure checkout, + /// so this measures the pool's acquire/return path under concurrency. /// + /// + /// + /// Because nothing happens between checkout and return, the tasks stay phase-locked and + /// the idle channel oscillates around empty even though the pool is full. That makes this + /// benchmark unusually sensitive to how a returned connection is handed to a waiting + /// caller: under the V2 pool an inline TryRead miss parks the caller in + /// ReadAsync, so it resumes on a threadpool continuation. Read it as a + /// zero-hold-time worst case for wake-up scheduling, not as typical application + /// behaviour: real callers do some work while holding a connection, which decorrelates + /// returns from checkouts and lets the fast path hit. + /// + /// + /// Compare against for the same loop with no + /// concurrency, and for concurrency with a + /// realistic hold time. A regression here alongside flat or improved results in those two + /// indicates a change in wake-up scheduling rather than in checkout cost. + /// + /// [Benchmark] - public async Task RapidFireOpenClose() + public async Task RapidFireOpenCloseAsync() { - int iterationsPerTask = Math.Max(20, MaxPoolSize / Math.Max(1, Parallelism) * 4); + int iterationsPerTask = RapidFireCheckouts / Parallelism; var tasks = new Task[Parallelism]; for (int i = 0; i < Parallelism; i++) { @@ -103,6 +196,43 @@ public async Task RapidFireOpenClose() await Task.WhenAll(tasks); } + /// + /// Sync counterpart to : the same zero-hold churn, but + /// every checkout goes through the blocking Open() path. + /// + /// + /// Worth measuring separately because the two paths diverge inside the pool. The V2 pool + /// has no synchronous channel read, so a sync caller that misses the inline fast path has + /// to run the async wait synchronously, which is a materially different cost from + /// awaiting it. At 10 the pool has idle spares throughout, so + /// that cell measures concurrent sync checkout on the fast path. At Parallelism 50 with + /// 50 the pool is fully subscribed and the zero-hold loop can + /// drain the idle channel, so that cell is this class's coverage of the synchronous + /// wake path. + /// + /// Workers run on threadpool threads deliberately: sync database calls in ASP.NET run + /// there, so that is the configuration whose behaviour actually matters. + /// + [Benchmark] + public async Task RapidFireOpenCloseSync() + { + int iterationsPerTask = RapidFireCheckouts / Parallelism; + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Run(() => + { + for (int j = 0; j < iterationsPerTask; j++) + { + using var conn = new SqlConnection(_connectionString); + conn.Open(); + // immediate return to pool + } + }); + } + await Task.WhenAll(tasks); + } + /// /// Randomized hold — each task opens a connection, holds it for a random duration /// (0-50ms), optionally runs a query, then returns it. Simulates realistic mixed @@ -214,11 +344,18 @@ public async Task MultiCommandReuse() /// wait. Measures how well the pool handles back-pressure when all connections are /// checked out and callers are queued. /// + /// + /// sets how far the pool is oversubscribed, and so how deep + /// the queue of waiting callers gets. It is the queue depth rather than the concurrency + /// level here, because saturating the pool already takes + /// tasks before any caller has to wait at all. + /// [Benchmark] public async Task PoolExhaustionRecovery() { - // Ensure we exceed pool capacity - int taskCount = Math.Max(Parallelism, MaxPoolSize * 2); + // Saturate the pool, then oversubscribe it by Parallelism so exactly that many + // callers are queued waiting for a connection to come back. + int taskCount = MaxPoolSize + Parallelism; var tasks = new Task[taskCount]; for (int i = 0; i < taskCount; i++) { diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolThreadPoolPressureRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolThreadPoolPressureRunner.cs new file mode 100644 index 0000000000..d898800da9 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolThreadPoolPressureRunner.cs @@ -0,0 +1,256 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Data.SqlClient.PerformanceTests +{ + /// + /// Measures a saturated pool driven by sync callers on threadpool threads, with the + /// threadpool's minimum worker count pinned so the result is reproducible. + /// + /// A sync Open() against a saturated pool blocks its thread. When those threads + /// are threadpool threads, a pool whose waiter wake-up requires a queued continuation + /// cannot make progress: every thread is blocked in a wait, so the wake-up sits in the + /// queue until the threadpool injects another thread. On the TFMs this project builds the + /// runtime is told about cooperative blocking and compensates quickly, so stalls are tens + /// to a few hundred milliseconds. net462 has no equivalent notification and is slower, but + /// this suite does not build it, so that path is not measured here. + /// + /// covers the same shape at the default + /// threadpool floor, which makes it dependent on hill-climbing timing and therefore + /// noisy. Pinning the floor turns that into a controlled comparison: + /// + /// - below guarantees the + /// threadpool starts starved, so the wake path is exercised on every run. + /// - above pre-creates enough + /// threads that injection never gates progress. This is the control: a pool that only + /// regresses in the starved configuration has a wake-path problem, not a throughput + /// problem. + /// + /// The effect is tail latency, not a shifted median, so compare distributions rather + /// than means alone. + /// + /// A regression here is largely a statement about application configuration rather than a + /// pool defect, but the starved configuration is not exotic: the default minimum is + /// , which honours cgroup CPU quotas, so a service + /// in a 1-2 vCPU container runs with a floor of 1 or 2. This runner exists to characterise + /// where that boundary is and to catch it moving, not to drive the delta to zero. + /// + /// The pool implementation (legacy vs V2) is a process-level choice - see the remarks on + /// . Run twice (UseConnectionPoolV2 false then + /// true) to compare. + /// + /// Related issue: #3356 + /// + public class ConnectionPoolThreadPoolPressureRunner : BaseRunner + { + /// + /// Number of concurrent sync workers. Also the reference point for the non-starved + /// control in , so it is a constant rather than a + /// literal on the attribute. + /// + private const int WorkerParallelism = 50; + + /// + /// Number of concurrent sync workers, all running on threadpool threads. + /// + [Params(WorkerParallelism)] + public int Parallelism { get; set; } + + /// + /// Max pool size. Deliberately smaller than so most + /// workers must block waiting for a connection to be returned. Without that + /// back-pressure nobody waits and the wake path is never exercised. + /// + [Params(10)] + public int MaxPoolSize { get; set; } + + /// + /// Threadpool minimum worker thread count, pinned for the duration of the run. + /// + /// + /// Sourced from rather than fixed constants, + /// because this number is only meaningful relative to + /// — see that property for why. + /// + [ParamsSource(nameof(MinWorkerThreadsValues))] + public int MinWorkerThreads { get; set; } + + /// + /// The minimum worker counts to sweep, expressed as multiples of + /// . + /// + /// + /// Fixed constants would not survive a change of machine. The processor count is both + /// the value this parameter displaces (it is the runtime's own default floor) and the + /// size of the runtime's immediate cooperative-blocking injection budget, so the same + /// absolute number means "starved" on one host and "generous" on another. A constant + /// chosen to starve a 16-core benchmark machine would quietly stop starving anything on + /// a 4-core developer box, and the benchmark would keep reporting numbers that no longer + /// measure the wake path. + /// + /// The multiples map onto real deployments: a quarter of the processor count stands in + /// for the default floor of a 2-4 vCPU container, 1x is the runtime default that almost + /// every application actually runs, and 2x is the most common explicit multiplier in + /// shipped code. + /// + /// The control is the exception: it is pinned to twice + /// rather than a processor multiple, because a processor multiple is not guaranteed to + /// clear it. On a 4-core host ProcessorCount * 8 is 32, below the 50 workers, so + /// every case would be starved and the sweep would have no non-starved baseline to + /// compare against. Taking the larger of the two keeps the processor-relative scaling on + /// hosts where it is already sufficient. + /// + /// Floored at 1 because SetMinThreads rejects 0, and de-duplicated because the + /// lower multiples collapse together on very small hosts. + /// + public static IEnumerable MinWorkerThreadsValues => + new[] + { + Environment.ProcessorCount / 4, + Environment.ProcessorCount, + Environment.ProcessorCount * 2, + Math.Max(Environment.ProcessorCount * 8, WorkerParallelism * 2), + } + .Select(static value => Math.Max(1, value)) + .Distinct(); + + /// + /// Number of open/query/close operations each worker performs per invocation. + /// + [Params(20)] + public int OpsPerWorker { get; set; } + + private string _connectionString; + private int _originalMinWorkerThreads; + private int _originalMinCompletionPortThreads; + + [GlobalSetup] + public void Setup() + { + Console.WriteLine( + "[ConnectionPoolThreadPoolPressureRunner] Pool implementation: " + + (s_config.UseConnectionPoolV2 + ? "ChannelDbConnectionPool (V2)" + : "WaitHandleDbConnectionPool (legacy)")); + + ThreadPool.GetMinThreads( + out _originalMinWorkerThreads, out _originalMinCompletionPortThreads); + SetMinWorkerThreads(MinWorkerThreads, _originalMinCompletionPortThreads); + + var builder = new SqlConnectionStringBuilder(s_config.ConnectionString) + { + Pooling = true, + MaxPoolSize = MaxPoolSize, + MinPoolSize = 0 + }; + _connectionString = builder.ConnectionString; + } + + [GlobalCleanup] + public void Cleanup() + { + // Restoring matters beyond this benchmark: the floor is process-wide, so leaving it + // raised would silently change the conditions for every runner that follows in the + // same process. + SetMinWorkerThreads( + _originalMinWorkerThreads, _originalMinCompletionPortThreads); + } + + /// + /// Pins the threadpool worker floor, failing loudly if it does not take effect. + /// + /// is the only variable this benchmark manipulates, so a + /// refused or clamped request would leave every configuration running at the same floor + /// and produce a comparison that looks valid but measures nothing. The return value alone + /// is not sufficient evidence, so the value is also read back. + /// + private static void SetMinWorkerThreads(int workerThreads, int completionPortThreads) + { + if (!ThreadPool.SetMinThreads(workerThreads, completionPortThreads)) + { + throw new InvalidOperationException( + $"ThreadPool.SetMinThreads({workerThreads}, {completionPortThreads}) was refused. " + + "The threadpool floor is this benchmark's independent variable, so the run would " + + "otherwise report a comparison that did not actually vary it."); + } + + ThreadPool.GetMinThreads(out int actualWorkerThreads, out _); + if (actualWorkerThreads != workerThreads) + { + throw new InvalidOperationException( + $"ThreadPool.SetMinThreads({workerThreads}, ...) reported success but the floor " + + $"read back as {actualWorkerThreads}. The runtime clamped the request, so this " + + "configuration would not measure the intended threadpool pressure."); + } + } + + [IterationSetup] + public void IterationSetup() + { + // Warm the pool to MaxPoolSize so the measured run reflects steady-state + // checkout/return rather than first-time physical connection establishment. + WarmPool(MaxPoolSize); + } + + [IterationCleanup] + public void IterationCleanup() + { + using var conn = new SqlConnection(_connectionString); + SqlConnection.ClearPool(conn); + } + + [Benchmark] + public Task SaturatedSyncOpenOnThreadPool() + { + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Run(() => + { + for (int op = 0; op < OpsPerWorker; op++) + { + using var conn = new SqlConnection(_connectionString); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT 1"; + _ = cmd.ExecuteScalar(); + // Dispose returns the connection to the pool. + } + }); + } + + return Task.WhenAll(tasks); + } + + private void WarmPool(int count) + { + var conns = new SqlConnection[count]; + try + { + for (int i = 0; i < count; i++) + { + conns[i] = new SqlConnection(_connectionString); + conns[i].Open(); + } + } + finally + { + // Close and dispose every connection so they return to the pool and + // are not retained until GC (which would add allocation/GC noise). + for (int i = 0; i < count; i++) + { + conns[i]?.Close(); + conns[i]?.Dispose(); + } + } + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs index 7b136c6f2c..41d756c200 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs @@ -71,6 +71,8 @@ public class Benchmarks public RunnerJob ConnectionPoolStressRunnerConfig; public RunnerJob ConnectionPoolContentionRunnerConfig; public RunnerJob ConnectionPoolChurnRunnerConfig; + public RunnerJob ConnectionPoolRampRunnerConfig; + public RunnerJob ConnectionPoolThreadPoolPressureRunnerConfig; } public class RunnerJob diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs index 5a0d303e97..e326766f29 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs @@ -55,6 +55,8 @@ public BenchmarkUnit(string name, Func selector, Type run new BenchmarkUnit("ConnectionPoolStress", b => b.ConnectionPoolStressRunnerConfig, typeof(ConnectionPoolStressRunner)), new BenchmarkUnit("ConnectionPoolContention", b => b.ConnectionPoolContentionRunnerConfig, typeof(ConnectionPoolContentionRunner)), new BenchmarkUnit("ConnectionPoolChurn", b => b.ConnectionPoolChurnRunnerConfig, typeof(ConnectionPoolChurnRunner)), + new BenchmarkUnit("ConnectionPoolRamp", b => b.ConnectionPoolRampRunnerConfig, typeof(ConnectionPoolRampRunner)), + new BenchmarkUnit("ConnectionPoolThreadPoolPressure", b => b.ConnectionPoolThreadPoolPressureRunnerConfig, typeof(ConnectionPoolThreadPoolPressureRunner)), }; /// diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc index 0ecc44e957..37bf73e960 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc @@ -173,6 +173,24 @@ "InvocationCount": 1, "WarmupCount": 1, "RowCount": 0 + }, + "ConnectionPoolRampRunnerConfig": { + "Enabled": true, + "LaunchCount": 1, + "IterationCount": 15, + "InvocationCount": 1, + "WarmupCount": 1, + "RowCount": 0 + }, + // Measures tail latency from threadpool starvation, so it needs more iterations + // than the others: a stall shows up in the distribution, not in the median. + "ConnectionPoolThreadPoolPressureRunnerConfig": { + "Enabled": true, + "LaunchCount": 1, + "IterationCount": 25, + "InvocationCount": 1, + "WarmupCount": 1, + "RowCount": 0 } } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolTest.cs index d647ab0914..01321a35a6 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolTest.cs @@ -235,6 +235,101 @@ out DbConnectionInternal? internalConnection Assert.Equal(pool.PoolGroupOptions.MaxPoolSize, pool.Count); } + /// + /// Verifies that an asynchronous request satisfied by an already-idle connection is + /// completed inline on the caller's thread, rather than being dispatched to the thread pool. + /// + /// + /// The fast path must report completion by returning true with the connection, exactly as + /// WaitHandleDbConnectionPool does on its inline hit. Completing the TaskCompletionSource + /// and returning false would look equivalent but is not: it sends + /// SqlConnection.InternalOpenAsync down its asynchronous branch, which allocates an + /// OpenAsyncRetry and schedules ContinueWith(..., TaskScheduler.Default), costing a thread + /// pool dispatch even though the result is already available. Asserting that the + /// TaskCompletionSource is left untouched is what pins that down. + /// + [Fact] + public void GetConnectionAsync_WithIdleConnection_ShouldCompleteInline() + { + // Arrange: take a connection and return it, leaving one connection idle in the pool. + var pool = ConstructPool(SuccessfulConnectionFactory); + SqlConnection owningConnection = new(); + + pool.TryGetConnection( + owningConnection, + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? pooledConnection + ); + Assert.NotNull(pooledConnection); + pool.ReturnInternalConnection(pooledConnection, owningConnection); + + // Act + TaskCompletionSource taskCompletionSource = new(); + SqlConnection secondOwner = new(); + var completed = pool.TryGetConnection( + secondOwner, + taskCompletionSource, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? internalConnection + ); + + // Assert: the request is reported as completed and the connection is handed back + // directly, matching WaitHandleDbConnectionPool's inline hit. This is what lets + // SqlConnection.InternalOpenAsync take its synchronous branch and skip the + // OpenAsyncRetry allocation and the ContinueWith thread pool dispatch. + Assert.True(completed); + Assert.Equal(pooledConnection, internalConnection); + + // The fast path must also activate the connection and assign ownership, exactly as the + // full path does. Without this the pool would hand back an unowned, unactivated + // connection and the assertions above would still pass. + Assert.Same(secondOwner, internalConnection!.Owner); + + // The TaskCompletionSource must be left alone; the caller abandons it on a + // synchronous completion. + Assert.False(taskCompletionSource.Task.IsCompleted); + + // The idle connection was reused rather than a second one being opened. + Assert.Equal(1, pool.Count); + } + + /// + /// Verifies that a synchronous request satisfied by an already-idle connection returns that + /// connection inline. + /// + [Fact] + public void GetConnection_WithIdleConnection_ShouldReturnInline() + { + // Arrange: take a connection and return it, leaving one connection idle in the pool. + var pool = ConstructPool(SuccessfulConnectionFactory); + SqlConnection owningConnection = new(); + + pool.TryGetConnection( + owningConnection, + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? pooledConnection + ); + Assert.NotNull(pooledConnection); + pool.ReturnInternalConnection(pooledConnection, owningConnection); + + // Act + SqlConnection secondOwner = new(); + var completed = pool.TryGetConnection( + secondOwner, + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? internalConnection + ); + + // Assert + Assert.True(completed); + Assert.Equal(pooledConnection, internalConnection); + Assert.Same(secondOwner, internalConnection!.Owner); + Assert.Equal(1, pool.Count); + } + /// /// Verifies that a waiting synchronous caller reuses a connection that is returned to an /// exhausted pool instead of creating a new physical connection. @@ -652,10 +747,19 @@ public void StressTestAsync() TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? internalConnection ); - internalConnection = await taskCompletionSource.Task; - pool.ReturnInternalConnection(internalConnection, owningObject); + + // A request satisfied from the pool's existing connections completes inline, + // returning the connection directly and leaving the TaskCompletionSource + // untouched. Only fall back to awaiting it when the request was handed off. + // This mirrors how the pool's callers consume TryGetConnection. + if (!completed) + { + internalConnection = await taskCompletionSource.Task; + } Assert.NotNull(internalConnection); + + pool.ReturnInternalConnection(internalConnection, owningObject); }); tasks.Add(t); } @@ -2282,6 +2386,94 @@ protected override DbConnectionInternal CreateConnection( #endregion + #region Saturated Sync Wait Tests + + /// + /// Verifies that the saturated synchronous checkout path makes forward progress when every + /// waiter is blocked in ReadChannelSyncOverAsync on a threadpool thread. This is the + /// net462 guard for that path, where the wake-up gets no cooperative-blocking notification; + /// it asserts liveness only, since the frameworks legitimately differ on latency. + /// + [Fact] + public void SyncCheckout_WhenSaturatedOnThreadPoolThreads_AllWaitersMakeProgress() + { + // Arrange + const int MaxPoolSize = 4; + ThreadPool.GetMinThreads(out int minWorker, out _); + + // More blocked waiters than the pool can serve AND than the threadpool floor, so the + // wake path is exercised rather than every worker simply getting its own thread. + // Deliberately uncapped: any ceiling would silently stop saturating on a host whose + // floor already exceeds it, turning this into a no-op exactly where it matters most. + int workerCount = minWorker + MaxPoolSize + 8; + + var poolGroupOptions = new DbConnectionPoolGroupOptions( + poolByIdentity: false, + minPoolSize: 0, + maxPoolSize: MaxPoolSize, + // Generous: a healthy run finishes far inside this, while a genuinely stuck wake + // path still fails the test rather than hanging the suite forever. + creationTimeout: 60, + loadBalanceTimeout: 0, + hasTransactionAffinity: true, + idleTimeout: 0); + var pool = ConstructPool(SuccessfulConnectionFactory, poolGroupOptions: poolGroupOptions); + + using var startGate = new ManualResetEventSlim(initialState: false); + var acquired = new ConcurrentBag(); + var failures = new ConcurrentBag(); + var workers = new Task[workerCount]; + + // Act: release every worker at once so they pile onto the pool together. + for (int i = 0; i < workerCount; i++) + { + workers[i] = Task.Factory.StartNew( + () => + { + try + { + startGate.Wait(); + bool got = pool.TryGetConnection( + new SqlConnection(), + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(60)), + out DbConnectionInternal? connection); + + acquired.Add(got && connection is not null); + + // Return promptly: with MaxPoolSize connections shared by every worker, + // each return is what wakes the next waiter. + if (connection is not null) + { + pool.ReturnInternalConnection(connection, connection.Owner); + } + } + catch (Exception ex) + { + failures.Add(ex); + } + }, + CancellationToken.None, + // LongRunning would hand each worker a dedicated thread, which is precisely the + // starvation this test needs to reproduce. Keep them on threadpool threads. + TaskCreationOptions.None, + TaskScheduler.Default); + } + + startGate.Set(); + + // Assert + Assert.True( + Task.WaitAll(workers, TimeSpan.FromSeconds(120)), + $"Saturated sync checkout did not drain: {acquired.Count} of {workerCount} workers " + + "finished. The waiter wake path is not making progress on this framework."); + Assert.Empty(failures); + Assert.Equal(workerCount, acquired.Count); + Assert.DoesNotContain(false, acquired); + } + + #endregion + #region Connection Timeout Awareness Tests /// From faa131ea6f405511b4cf98d4e3f99e2c697eadf1 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Fri, 4 Sep 2026 08:27:56 -0300 Subject: [PATCH 15/51] Enable SDL Roslyn analysis (incl. internal IA* analyzers) in the OneBranch build (#4464) --- .config/guardian/.gdnbaselines | 669 +++++++++++++++++- Directory.Packages.props | 18 + NuGet.analysis.config | 50 ++ NuGet.config | 42 +- build.proj | 122 ++++ .../templates/steps/override-sni-version.yml | 13 + .../onebranch/jobs/build-buildproj-job.yml | 4 +- .../onebranch/sqlclient-non-official.yml | 60 +- .../onebranch/sqlclient-official.yml | 59 +- .../steps/roslyn-analyzers-buildproj-step.yml | 171 ++++- .../variables/onebranch-variables.yml | 13 +- src/Directory.Build.props | 65 ++ src/Directory.Build.targets | 21 + .../Microsoft.Data.SqlClient.csproj | 6 +- .../TDS/TDS.EndPoint/TDS.EndPoint.csproj | 1 - tools/PackageCompatibility/NuGet.config | 35 + 16 files changed, 1306 insertions(+), 43 deletions(-) create mode 100644 NuGet.analysis.config create mode 100644 src/Directory.Build.targets diff --git a/.config/guardian/.gdnbaselines b/.config/guardian/.gdnbaselines index 10698219a1..e5c5ff89b2 100644 --- a/.config/guardian/.gdnbaselines +++ b/.config/guardian/.gdnbaselines @@ -8,7 +8,7 @@ "default": { "name": "default", "createdDate": "2026-07-23 11:29:23Z", - "lastUpdatedDate": "2026-07-23 11:29:23Z" + "lastUpdatedDate": "2026-08-28 14:33:24Z" } }, "results": { @@ -103,10 +103,15 @@ "0d5b851e97bdb0eb8931e6f326718a657f9cd46092eb8a2a2d414be2147a797c", "e6c0cd6ef2433a42c95a2939cce740019ecda3fcfde64a3a0f21661e6ff27f71" ], + "target": "src/Microsoft.Data.SqlClient/tests/ManualTests/makepfxcert.ps1", + "line": 145, + "uriBaseId": "file:///D:/a/_work/1/s/", "memberOf": [ "default" ], - "createdDate": "2026-07-23 11:29:23Z" + "tool": "psscriptanalyzer", + "ruleId": "PSAvoidUsingConvertToSecureStringWithPlainText", + "createdDate": "2026-08-28 12:58:19Z" }, "27cf35f7df3f630fab489573ec19318f563e042424ca30625e9fe08407d74bdf": { "signature": "27cf35f7df3f630fab489573ec19318f563e042424ca30625e9fe08407d74bdf", @@ -119,6 +124,664 @@ "default" ], "createdDate": "2026-07-23 11:29:23Z" + }, + "1e0989a7cdd65afb10dd3787a2ed33e9f737e6dd049524edbf76ba1daadf6ef8": { + "signature": "1e0989a7cdd65afb10dd3787a2ed33e9f737e6dd049524edbf76ba1daadf6ef8", + "alternativeSignatures": [ + "47067564034219f2cf40604fcd1beadb34ebe1b960cc75600796c8a0f4565604" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/src/Utils.cs", + "line": 72, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 14:17:20Z" + }, + "e5cd66384b36741191c98844947e6b857140c09b2c352b180664f8b4829c3e4c": { + "signature": "e5cd66384b36741191c98844947e6b857140c09b2c352b180664f8b4829c3e4c", + "alternativeSignatures": [ + "07fc741e29b6f1d01d84d3290b8c53dc7f22036a8313d1f41acdca253aa3e254" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Resources/StringsHelper.cs", + "line": 90, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "281a106076dd3d70aefcd230a90cef542f52488fb3ce164c94b213ededa12da5": { + "signature": "281a106076dd3d70aefcd230a90cef542f52488fb3ce164c94b213ededa12da5", + "alternativeSignatures": [ + "42cf7833cc5d63447162200f8926b57746ad3f6f2bd43318f0e606f022933c3e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/AzureAttestationBasedEnclaveProvider.cs", + "line": 215, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "fcda2db01b63d59f5a4aa73cb780405887f4616f26f9e23dfae52195cab4994e": { + "signature": "fcda2db01b63d59f5a4aa73cb780405887f4616f26f9e23dfae52195cab4994e", + "alternativeSignatures": [ + "9134dead4de902cc02f5f2e7785d84b422f31847f237ba6bcbaeb823e228fd7d" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveProviderBase.cs", + "line": 170, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "4ff2eb551fd17b4d3239b89cf97bfb09507cdfb631cb4787adc3a4f195e8bac7": { + "signature": "4ff2eb551fd17b4d3239b89cf97bfb09507cdfb631cb4787adc3a4f195e8bac7", + "alternativeSignatures": [ + "3b7985cbb5123ba5b91edc3e36a952d1f9d579943820f3c3e870095c7e264cc4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/NoneAttestationEnclaveProvider.cs", + "line": 43, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8f3e9755a800f590583d9e7ea2028c5d23ee2008450af7d55daea17d5fbecfa5": { + "signature": "8f3e9755a800f590583d9e7ea2028c5d23ee2008450af7d55daea17d5fbecfa5", + "alternativeSignatures": [ + "4531f356921a98edacfca7c32eb389b459c02014ea3db2d2d899ce65a87d52ca" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlAeadAes256CbcHmac256EncryptionKey.cs", + "line": 94, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "6ba87c464ec17dc52c0304f5cee224f8d3233507a79211916901a7b9d0d0908c": { + "signature": "6ba87c464ec17dc52c0304f5cee224f8d3233507a79211916901a7b9d0d0908c", + "alternativeSignatures": [ + "fcd1d28e2fe4772861caa303138474dc79869cc77079f79b55eda1612a4c4693" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlAuthenticationProviderManager.cs", + "line": 353, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "ffe242f34f321ccf4d2e727c9812baaf006e4ad122d314e02dd287f388b5b176": { + "signature": "ffe242f34f321ccf4d2e727c9812baaf006e4ad122d314e02dd287f388b5b176", + "alternativeSignatures": [ + "ca2f6e8136bafc65dc12eb6236123ee0877de7a1b48e810c36b7feae643b9654" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs", + "line": 1049, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "b73abd622849352bbe6c5188f106d4a2f9a1c4f650b7d2ba8f383653909a9479": { + "signature": "b73abd622849352bbe6c5188f106d4a2f9a1c4f650b7d2ba8f383653909a9479", + "alternativeSignatures": [ + "636e8fa98391919cfbd7f27792cac5c7806dbd7ec3ab0506b27fbbe52ee9cd8b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlClientPermission.netfx.cs", + "line": 144, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "cdbeec94358c58eb2960ea291f7f804e5d24e15807bc5b4a5ce259782727cd9a": { + "signature": "cdbeec94358c58eb2960ea291f7f804e5d24e15807bc5b4a5ce259782727cd9a", + "alternativeSignatures": [ + "8fa2f809347c794dbb3659a25cb2ea3a15df3a64a86f5a78f72b39c63f6b8319" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs", + "line": 2336, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "01c8d80a7268d44c7d8e478f887e804d8819cc2382ad1d845a415d97ead39d2e": { + "signature": "01c8d80a7268d44c7d8e478f887e804d8819cc2382ad1d845a415d97ead39d2e", + "alternativeSignatures": [ + "00aae68e847dfaed6240e67d9f0d1f5f64b9a0343c185c69f7dae148ebf2dc35" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionEncryptOption.cs", + "line": 57, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "5ef60ae6fdf13d9fcebe7631af27f7ea23d3f198215a19c8752bf5f8cdee8dc9": { + "signature": "5ef60ae6fdf13d9fcebe7631af27f7ea23d3f198215a19c8752bf5f8cdee8dc9", + "alternativeSignatures": [ + "17d62daf6be556a78b7a342be79f5038d7f1831851722a69398b996cecd57bd8" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionEncryptOption.cs", + "line": 108, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f99754da199aaa7a5e510552f0b4e851029c149dced1d5196eba374b03f0450d": { + "signature": "f99754da199aaa7a5e510552f0b4e851029c149dced1d5196eba374b03f0450d", + "alternativeSignatures": [ + "2852f83af1a5eacc738153cae383e7e216179f3573e9082879dddb48e32a260c" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs", + "line": 1638, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "3dc0ef1e00dd1aed1bd9b6a2e9c06c4f8b24bf368419a2928feaab51252a3b47": { + "signature": "3dc0ef1e00dd1aed1bd9b6a2e9c06c4f8b24bf368419a2928feaab51252a3b47", + "alternativeSignatures": [ + "6a46ad5f8328cb647c28327bb4260335dab6e381ee816f60e371f3f3c4f348b0" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs", + "line": 1640, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d0e489f06819d4a2e78e99be8aedb5d683f8c787cfc07d5c25689bc4e4e3f5f6": { + "signature": "d0e489f06819d4a2e78e99be8aedb5d683f8c787cfc07d5c25689bc4e4e3f5f6", + "alternativeSignatures": [ + "524203b79cc017dc30443712f932710574a9a39d5f7251a4c9354f3cdd21b36b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.Debug.cs", + "line": 59, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "6d1f24e834de7bf17aa69abdd0fe79751a4afcc083253e84b577c813a19f46fb": { + "signature": "6d1f24e834de7bf17aa69abdd0fe79751a4afcc083253e84b577c813a19f46fb", + "alternativeSignatures": [ + "9b805fc1d43486b21ec75d68dd2e4b19d5e7af7c954c546389307bd689a2930b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.Debug.cs", + "line": 59, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d245c0ad54d0229b6ea7a194bd42c40011b734ee5ef92dfce42c9a40096c906d": { + "signature": "d245c0ad54d0229b6ea7a194bd42c40011b734ee5ef92dfce42c9a40096c906d", + "alternativeSignatures": [ + "472b266989720cacdc2666a97234830e954f84346ba13ee028d9f0dbac3d5d82" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs", + "line": 2801, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "68fc925a125fdb6381d42edaaae658cb5c23c5bbef262cbb569215d839e3a9b3": { + "signature": "68fc925a125fdb6381d42edaaae658cb5c23c5bbef262cbb569215d839e3a9b3", + "alternativeSignatures": [ + "1a6475a1a8210fd0e0d4807a4c5d4e1017da257dbfa97b17c8e3382c1684a34a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDependency.cs", + "line": 644, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA2219", + "createdDate": "2026-08-28 13:55:54Z" + }, + "df9958d713606f4b2d800a8f5b33e4839cde7d9e7514f732e0b999f4e7df0cb0": { + "signature": "df9958d713606f4b2d800a8f5b33e4839cde7d9e7514f732e0b999f4e7df0cb0", + "alternativeSignatures": [ + "b44a5e32ae614b1bdda97a1a634dad5a4ab4ede29b463dd3196f309db10e1d15" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDependency.cs", + "line": 1222, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "e318e84cf65f756c454457b437010a21fac2f27f2fa7378c3befc809b97369be": { + "signature": "e318e84cf65f756c454457b437010a21fac2f27f2fa7378c3befc809b97369be", + "alternativeSignatures": [ + "d8118425a1409e87f5983a840ac22cc663a7ab494bb914b312f7b32adb84d5f5" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs", + "line": 1134, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "467568c2128c495889d899ef783ccca652f3b355aa8fe815d4da1b55b6deab95": { + "signature": "467568c2128c495889d899ef783ccca652f3b355aa8fe815d4da1b55b6deab95", + "alternativeSignatures": [ + "8962958e2e5e468cc56038a04d02476e352bf6f1d48104f51c13990d364b0c64" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlException.cs", + "line": 164, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "727c936839845e9a68ff00fbc69abb5a132769506d3cfe87692612a2addac855": { + "signature": "727c936839845e9a68ff00fbc69abb5a132769506d3cfe87692612a2addac855", + "alternativeSignatures": [ + "2450ed8367a7fac65b41524a145bf097f2dee5794752124fefa3a635057946bd" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs", + "line": 114, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8c6b44ec1f44fbe00469a48beb3f9eee61c8e31e237c53ba372a1abfc5ea481f": { + "signature": "8c6b44ec1f44fbe00469a48beb3f9eee61c8e31e237c53ba372a1abfc5ea481f", + "alternativeSignatures": [ + "7a3c05145c302720ca6feadabcacbb11303442b2289fb06796e21b713fd63717" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs", + "line": 572, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8a5592e024f45a9bdce3315e108cccfafe99b184dbcb4e50d59d783fa3db3942": { + "signature": "8a5592e024f45a9bdce3315e108cccfafe99b184dbcb4e50d59d783fa3db3942", + "alternativeSignatures": [ + "929389afe5818dcc5113299f0e0263eca26ef989785742ac7e46b088d5cae598" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs", + "line": 2364, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "770e1d8ef5a06c9b4b552dc9c8bf000f572d633bf11ded7e749b3ac9c07d208b": { + "signature": "770e1d8ef5a06c9b4b552dc9c8bf000f572d633bf11ded7e749b3ac9c07d208b", + "alternativeSignatures": [ + "4cd0e3d7087eaa0eaf05bec58d32fd71b660033f7bf3984392998d0b9946fb47" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlSecurityUtility.cs", + "line": 389, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8502b61241cdbfac98f7c84b8c16d4d4ec0ea2185018d9fce710e2bac445e8b0": { + "signature": "8502b61241cdbfac98f7c84b8c16d4d4ec0ea2185018d9fce710e2bac445e8b0", + "alternativeSignatures": [ + "afed061c02d7b602c516f09952197b58c6c1cdfa0e07ee3625af48287869b2c5" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs", + "line": 1749, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "a83b0d426733a42a34b4569643e60513b598035d6807fa8cda6f4a3501084929": { + "signature": "a83b0d426733a42a34b4569643e60513b598035d6807fa8cda6f4a3501084929", + "alternativeSignatures": [ + "3614422f0f09ab4fff1a0195a4f40acd53bf55e4781153ced9d736a801fc6aa4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs", + "line": 1875, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d36be8dd5189d09c4b5db9176bda7628839f0ba608f31ba449f1203ca9c30c09": { + "signature": "d36be8dd5189d09c4b5db9176bda7628839f0ba608f31ba449f1203ca9c30c09", + "alternativeSignatures": [ + "fcf7dc6efdfecd535087f2361d43cfa599cbe83b52717beb36c052dadbbc5a5d" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs", + "line": 2309, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "b19ca78715cc12a2051b38a495f41c060259e46d893be38e6dba447e8a87cc02": { + "signature": "b19ca78715cc12a2051b38a495f41c060259e46d893be38e6dba447e8a87cc02", + "alternativeSignatures": [ + "efc4d3f86b80b0d8392e7fa74b15ca51411078ed8f249b0f30911036530bc0b4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs", + "line": 3918, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "119b9514aeb84d5e34b5d7b1520378d4a2d017902d7d1bd572404f1b3186c59c": { + "signature": "119b9514aeb84d5e34b5d7b1520378d4a2d017902d7d1bd572404f1b3186c59c", + "alternativeSignatures": [ + "57389b1211c68b1c8ed21fb3439d1dd9bd8a8e25677ed63b1db38ff7bd3b0c6b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs", + "line": 4403, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "533d9dc25ef8183859aeed955b720d435e3035702b3b7c10d12204170815ec04": { + "signature": "533d9dc25ef8183859aeed955b720d435e3035702b3b7c10d12204170815ec04", + "alternativeSignatures": [ + "55c532f39c15ec069c540c661f1ccb7371ee8c03f74fad4515d9cd94db8bcca1" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserStateObjectNative.cs", + "line": 100, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "90dd8eeec55b68eaf94f971d1af06723d22efc0f7cb5fd15870765e6e6602ff9": { + "signature": "90dd8eeec55b68eaf94f971d1af06723d22efc0f7cb5fd15870765e6e6602ff9", + "alternativeSignatures": [ + "ac02e05713e85c6086fc30c51a7083a159329f0a222c9d0be513e6da9fda0300" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/VirtualSecureModeEnclaveProvider.cs", + "line": 84, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f1c711a9df14d19ac6682356d6648111a1e3c2ae051812e8f04a7466df045aa4": { + "signature": "f1c711a9df14d19ac6682356d6648111a1e3c2ae051812e8f04a7466df045aa4", + "alternativeSignatures": [ + "1e752a5b4246f05f58da2adde778eb3eb46238ad51a6a2013d3fd9d80aaf5422" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/VirtualSecureModeEnclaveProviderBase.cs", + "line": 488, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d159ab04bf10a650f1f43064602246acfa671e4f1f7ba07faf5620ab13043926": { + "signature": "d159ab04bf10a650f1f43064602246acfa671e4f1f7ba07faf5620ab13043926", + "alternativeSignatures": [ + "74e1a9e9972040959e4176eb8dc981d8394545eaf86f30e059f9814f545bea67" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionString.netfx.cs", + "line": 374, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "761ae24086cbfbeb9667e20ab45b174613bd156d0dc7bde56a37571db11cd779": { + "signature": "761ae24086cbfbeb9667e20ab45b174613bd156d0dc7bde56a37571db11cd779", + "alternativeSignatures": [ + "1793d6a0e1d5ee495ced5ce18af4bb1c6f1635ce89d6dfe9d840b84673ccbe7a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs", + "line": 4130, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "92759af0d8bc9a56339a09bfb5ae1e0adbe2c0aa7697911d0f14941ba65a1e67": { + "signature": "92759af0d8bc9a56339a09bfb5ae1e0adbe2c0aa7697911d0f14941ba65a1e67", + "alternativeSignatures": [ + "f20cbad7a6ed702f47f5d26f6e56844ce9f1e5a7d03152361b1818d4aef03058" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/DbConnectionPoolAuthenticationContextKey.cs", + "line": 83, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8409f840dce0e042b55250ce12045afaabc1e2e48f443e43ec0e493517e3e38d": { + "signature": "8409f840dce0e042b55250ce12045afaabc1e2e48f443e43ec0e493517e3e38d", + "alternativeSignatures": [ + "b607bb6c9aded6c9d9d1da95e9f49b9a17ec42880fe59156cfca1d2f088fec35" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniCommon.netcore.cs", + "line": 148, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "1152991aab5d089b6b086e389b075de228b319c82a577dae0afe850956202b12": { + "signature": "1152991aab5d089b6b086e389b075de228b319c82a577dae0afe850956202b12", + "alternativeSignatures": [ + "95a2b83edbb49524b5834cba58e1c1670ad5397cca2bc8ed4912f02c7b885acd" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs", + "line": 150, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "66052c0cd9b0dade1d5d86703650fa9c8d46f7977b60148aea67f371393e32b3": { + "signature": "66052c0cd9b0dade1d5d86703650fa9c8d46f7977b60148aea67f371393e32b3", + "alternativeSignatures": [ + "d67c06a92599b72202abeed718e385e4621947195fc3affc730a1066b55f5cb6" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs", + "line": 731, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f236b2c674fc8574908a9b254e10d9a78f4df86278d9cfe52f5b4d072689829b": { + "signature": "f236b2c674fc8574908a9b254e10d9a78f4df86278d9cfe52f5b4d072689829b", + "alternativeSignatures": [ + "fabdb2e276df6d043b029af9a40c831c8f9f7131dec81706dd664cd5d913f32e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniTcpHandle.netcore.cs", + "line": 658, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "424ec1510b6c1352e0731f19bd7a6c03f1da3bab869fa520046b0080cb7b5d70": { + "signature": "424ec1510b6c1352e0731f19bd7a6c03f1da3bab869fa520046b0080cb7b5d70", + "alternativeSignatures": [ + "4dac74c5b9483f0a4b1f7e76f91002aa485fd1b83b19d5823169b0c15eaace8b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SsrpClient.netcore.cs", + "line": 80, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "76482116b8cae39bdad9b92782ab216ff958f6578405f93879733326a4283bfe": { + "signature": "76482116b8cae39bdad9b92782ab216ff958f6578405f93879733326a4283bfe", + "alternativeSignatures": [ + "a815e32ff0ad176d42719f84cb015df61c89e9539fecbf58da169993e789b682" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs", + "line": 308, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "dd455bea13e4bd70696d1b0f4c890ca1ea42e18643bbe6b572989d84ac188f06": { + "signature": "dd455bea13e4bd70696d1b0f4c890ca1ea42e18643bbe6b572989d84ac188f06", + "alternativeSignatures": [ + "0444dab1ebebce0b0e0bfb453d8513e9e8108aad22055bd038c873a8a8bc1f0e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs", + "line": 629, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:36:06Z" + }, + "09526af76b9a6ad21a4841ea50bb51e72925789c5f0f733650ac946cc44060a9": { + "signature": "09526af76b9a6ad21a4841ea50bb51e72925789c5f0f733650ac946cc44060a9", + "alternativeSignatures": [ + "2a3fb00242d533ee3253b91c42ba7882b9c35ada1635e3470c481c6b43baa0e6" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/SqlUserDefinedAggregateAttribute.netstandard.cs", + "line": 61, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "6fec84f5f9e3087c484965319896c91b033557b8bcef7761b0ae74e1baf0ad7d": { + "signature": "6fec84f5f9e3087c484965319896c91b033557b8bcef7761b0ae74e1baf0ad7d", + "alternativeSignatures": [ + "efa9c07656c41d1d2367a89a7146fda28f3b584e2654693b4954d40bfafecb8f" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/SqlUserDefinedTypeAttribute.netstandard.cs", + "line": 73, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "1de5740b9e122c2d8bda4bfe66c94764a6192376f753a7234bac91e1fc28e5f6": { + "signature": "1de5740b9e122c2d8bda4bfe66c94764a6192376f753a7234bac91e1fc28e5f6", + "alternativeSignatures": [ + "14c7177139597c2ab94cf632b76bc03c4f2c252691b30a7ba69957e047b400c4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/StringsHelper.netstandard.cs", + "line": 130, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "0ca59be802da38e82370950b24300deae99834aa9a3cee38e064d1f17c5942a9": { + "signature": "0ca59be802da38e82370950b24300deae99834aa9a3cee38e064d1f17c5942a9", + "alternativeSignatures": [ + "26425ecc80bd3289865ce79ae6471c4ab21cde25acc1cf2067717fbf6d600a6a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.Internal/Logging/src/SqlClientEventSource.cs", + "line": 1995, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:09:32Z" } } -} \ No newline at end of file +} diff --git a/Directory.Packages.props b/Directory.Packages.props index 03ad924f6d..e28a1301a5 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -108,6 +108,24 @@ + + + + + + + diff --git a/NuGet.analysis.config b/NuGet.analysis.config new file mode 100644 index 0000000000..1ba52a93d3 --- /dev/null +++ b/NuGet.analysis.config @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/NuGet.config b/NuGet.config index a16ff70302..5bdc9722d5 100644 --- a/NuGet.config +++ b/NuGet.config @@ -5,19 +5,43 @@ + + + - + + + + + + + + + + + - - + + + + + + + + + + diff --git a/build.proj b/build.proj index c29dec53a1..31795fc6a7 100644 --- a/build.proj +++ b/build.proj @@ -176,6 +176,112 @@ --> false + + + + --no-incremental + -p:ArtifactPath="$(IsolatedBuildPath)/bin/" + + + + false + + -p:EnableAnalyzers=true + + + + false + $(EnableAnalyzersArgument) + -p:InternalAnalyzers=true + + + + $(EnableAnalyzersArgument) + -p:RestoreConfigFile="$(InternalAnalyzersNugetConfig)" + + + + $(EnableAnalyzersArgument) + -p:InternalAnalyzersVersion=$(InternalAnalyzersVersion) + + @@ -518,6 +628,8 @@ "$(DotnetPath)dotnet" build $(SqlClientProjectPath) -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) @@ -719,6 +831,8 @@ "$(DotnetPath)dotnet" build "$(AkvProviderProjectPath)" -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) @@ -824,6 +938,8 @@ "$(DotnetPath)dotnet" build "$(AbstractionsProjectPath)" -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) @@ -927,6 +1043,8 @@ "$(DotnetPath)dotnet" build "$(AzureProjectPath)" -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) @@ -1027,6 +1145,8 @@ "$(DotnetPath)dotnet" build $(LoggingProjectPath) -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) @@ -1091,6 +1211,8 @@ "$(DotnetPath)dotnet" build $(SqlServerProjectPath) -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) diff --git a/eng/pipelines/common/templates/steps/override-sni-version.yml b/eng/pipelines/common/templates/steps/override-sni-version.yml index 3b275262c3..b3496b9f1f 100644 --- a/eng/pipelines/common/templates/steps/override-sni-version.yml +++ b/eng/pipelines/common/templates/steps/override-sni-version.yml @@ -42,6 +42,19 @@ steps: # add the new package source $packageSources.AppendChild($newSource) + # Exact mappings take precedence over the governed feed's wildcard, ensuring validation SNI + # packages are restored from this source. Both package IDs are externally produced and are + # therefore intentionally not eligible for the repository's local feed. + $packageSourceMapping = $xml.SelectSingleNode('//ns:packageSourceMapping', $nsm) + $newMapping = $xml.CreateElement("packageSource") + $newMapping.SetAttribute("key","SNIValidation") + foreach ($packageId in @("Microsoft.Data.SqlClient.SNI", "Microsoft.Data.SqlClient.SNI.runtime")) { + $package = $xml.CreateElement("package") + $package.SetAttribute("pattern", $packageId) + $newMapping.AppendChild($package) + } + $packageSourceMapping.AppendChild($newMapping) + # save the xml file $xml.Save($NugetCfg) type $NugetCfg diff --git a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml index f773ec5e7e..d60ed634c0 100644 --- a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml +++ b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml @@ -149,7 +149,9 @@ jobs: # such as _CheckPwshToolRestored that run during RoslynAnalyzers and Build. - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - # Perform Roslyn analysis before building, since this step will clobber build output. + # Run Roslyn analysis. This step is self-contained: it performs its own build into an + # isolated output location, so it can run at any point in the job without clobbering the + # real build output below. - template: /eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml@self parameters: dependencyArguments: $(sqlServerVersionArgument) diff --git a/eng/pipelines/onebranch/sqlclient-non-official.yml b/eng/pipelines/onebranch/sqlclient-non-official.yml index 7eb7bfc6cc..6316c9654e 100644 --- a/eng/pipelines/onebranch/sqlclient-non-official.yml +++ b/eng/pipelines/onebranch/sqlclient-non-official.yml @@ -94,8 +94,29 @@ extends: parameters: featureFlags: + # WindowsHostVersion selects the Windows *host VM* that our Windows build container runs on. + # This is a separate layer from the container image itself (WindowsContainerImage in + # onebranch-variables.yml): the host is the outer machine running the Docker engine, and the + # container is where our build steps actually execute. + # + # These two must be kept compatible. Windows containers can only run on a host whose OS + # version is compatible with the container's base image. A mismatch (e.g. a 2025 container on + # a 2022 host) fails to start the container unless Hyper-V isolation is forced. + WindowsHostVersion: + Version: 2025 + + # We do NOT set a LinuxHostVersion. Unlike Windows, Linux containers share the host kernel, + # so there is no host/container OS-version compatibility requirement to satisfy -- our Linux + # build image (LinuxContainerImage in onebranch-variables.yml) runs on the default OneBranch + # Linux host regardless of its distribution. + + # CDPx is OneBranch's predecessor build system. When EnableCDPxPAT is true (the OneBranch + # default), the governed templates inject a legacy CDPx Personal Access Token and its + # associated NuGet / Azure Artifacts authentication variables (CDP_DEFAULT_CLIENT_PAT, + # VSS_NUGET_ACCESSTOKEN, VSS_NUGET_URI_PREFIXES, etc.) into the build and Docker jobs so + # package restore against Azure DevOps feeds works without explicit auth. We don't rely on + # that legacy CDPx package-authentication path, so we disable it. EnableCDPxPAT: false - WindowsHostVersion: 1ESWindows2022 release: # This indicates the pipeline category to deploy Box products. See: @@ -119,6 +140,34 @@ extends: # globalSdl: + # BREAK SEVERITY + # + # The SDL analyzer tasks never fail on findings; they only fail if the tool itself crashes or + # is misconfigured. The build break comes from the Post Analysis (Guardian Break) task, which + # reads the tool logs and fails when a finding meets or exceeds a minimum severity threshold. + # See https://aka.ms/gdn-azdo-break. + # + # Guardian normalises every finding to Error, Warning or Note. The threshold is cumulative, + # so a lower name is STRICTER, not looser: + # + # Error break on Error <-- Guardian's default + # Warning break on Error + Warning + # Note break on Error + Warning + Note + # + # Two knobs control it, in increasing order of precedence: + # + # globalSdl.severity threshold for every tool (maps to GdnBreakPolicyMinSev) + # globalSdl..severity per-tool override of the global threshold + # ob_sdl__severity per-job variable; overrides both of the above + # + # We omit `severity` everywhere and keep the Error-only default. OneBranch accepts ONLY + # Error, Warning or Note at this layer -- there is no explicit "Default" value to write, so + # inheriting the default requires omitting the key. Consequently, spelling out + # `severity: Error` on a tool is NOT equivalent to omitting it: it pins that tool to Error + # even if globalSdl.severity is later tightened. + # + # https://eng.ms/docs/products/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows/customizesdlforcontainerbuilds + # Snapshot of the SDL analyzer findings that pre-existed the breakOnSdlError rollout, so # builds only break on NEW findings. Generated from the SDL analysis artifacts of a full # non-official run and kept under .config/ alongside the other SDL tool configs @@ -198,7 +247,13 @@ extends: break: ${{ parameters.breakOnSdlError }} roslyn: - # Note, requires RoslynAnalyzers task to be added as a separate step + # Enabling Roslyn SDL analysis here requires that our .NET builds _produce_ Roslyn findings. + # You will see this in the separate Roslyn build task. + # + # Note that the Roslyn-specific Guardian collector/sanitizer requires SARIF v1, so our + # analysis build deliberately emits v1. Other, generic Guardian tooling expects SARIF v2 and + # may log processing errors (for example, Post Analysis's SDL artifact report) even though + # Roslyn collection and Guardian policy ingestion succeed. enabled: true break: ${{ parameters.breakOnSdlError }} @@ -235,6 +290,7 @@ extends: # TSA here does not by itself force breaking -- breakOnSdlError is what controls whether # findings fail the build. enabled: false + # Keep this in sync with Official even though TSA is disabled here. configFile: '$(REPO_ROOT)/.config/tsaoptions.json' stages: diff --git a/eng/pipelines/onebranch/sqlclient-official.yml b/eng/pipelines/onebranch/sqlclient-official.yml index 38a8a75496..00ec4f3f74 100644 --- a/eng/pipelines/onebranch/sqlclient-official.yml +++ b/eng/pipelines/onebranch/sqlclient-official.yml @@ -108,8 +108,29 @@ extends: parameters: featureFlags: + # WindowsHostVersion selects the Windows *host VM* that our Windows build container runs on. + # This is a separate layer from the container image itself (WindowsContainerImage in + # onebranch-variables.yml): the host is the outer machine running the Docker engine, and the + # container is where our build steps actually execute. + # + # These two must be kept compatible. Windows containers can only run on a host whose OS + # version is compatible with the container's base image. A mismatch (e.g. a 2025 container on + # a 2022 host) fails to start the container unless Hyper-V isolation is forced. + WindowsHostVersion: + Version: 2025 + + # We do NOT set a LinuxHostVersion. Unlike Windows, Linux containers share the host kernel, + # so there is no host/container OS-version compatibility requirement to satisfy -- our Linux + # build image (LinuxContainerImage in onebranch-variables.yml) runs on the default OneBranch + # Linux host regardless of its distribution. + + # CDPx is OneBranch's predecessor build system. When EnableCDPxPAT is true (the OneBranch + # default), the governed templates inject a legacy CDPx Personal Access Token and its + # associated NuGet / Azure Artifacts authentication variables (CDP_DEFAULT_CLIENT_PAT, + # VSS_NUGET_ACCESSTOKEN, VSS_NUGET_URI_PREFIXES, etc.) into the build and Docker jobs so + # package restore against Azure DevOps feeds works without explicit auth. We don't rely on + # that legacy CDPx package-authentication path, so we disable it. EnableCDPxPAT: false - WindowsHostVersion: 1ESWindows2022 release: # This indicates the pipeline category to deploy Box products. See: @@ -133,6 +154,34 @@ extends: # globalSdl: + # BREAK SEVERITY + # + # The SDL analyzer tasks never fail on findings; they only fail if the tool itself crashes or + # is misconfigured. The build break comes from the Post Analysis (Guardian Break) task, which + # reads the tool logs and fails when a finding meets or exceeds a minimum severity threshold. + # See https://aka.ms/gdn-azdo-break. + # + # Guardian normalises every finding to Error, Warning or Note. The threshold is cumulative, + # so a lower name is STRICTER, not looser: + # + # Error break on Error <-- Guardian's default + # Warning break on Error + Warning + # Note break on Error + Warning + Note + # + # Two knobs control it, in increasing order of precedence: + # + # globalSdl.severity threshold for every tool (maps to GdnBreakPolicyMinSev) + # globalSdl..severity per-tool override of the global threshold + # ob_sdl__severity per-job variable; overrides both of the above + # + # We omit `severity` everywhere and keep the Error-only default. OneBranch accepts ONLY + # Error, Warning or Note at this layer -- there is no explicit "Default" value to write, so + # inheriting the default requires omitting the key. Consequently, spelling out + # `severity: Error` on a tool is NOT equivalent to omitting it: it pins that tool to Error + # even if globalSdl.severity is later tightened. + # + # https://eng.ms/docs/products/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows/customizesdlforcontainerbuilds + # Snapshot of the SDL analyzer findings that pre-existed the breakOnSdlError rollout, so # builds only break on NEW findings. Generated from the SDL analysis artifacts of a full # non-official run and kept under .config/ alongside the other SDL tool configs @@ -212,7 +261,13 @@ extends: break: ${{ parameters.breakOnSdlError }} roslyn: - # Note, requires RoslynAnalyzers task to be added as a separate step + # Enabling Roslyn SDL analysis here requires that our .NET builds _produce_ Roslyn findings. + # You will see this in the separate Roslyn build task. + # + # Note that the Roslyn-specific Guardian collector/sanitizer requires SARIF v1, so our + # analysis build deliberately emits v1. Other, generic Guardian tooling expects SARIF v2 and + # may log processing errors (for example, Post Analysis's SDL artifact report) even though + # Roslyn collection and Guardian policy ingestion succeed. enabled: true break: ${{ parameters.breakOnSdlError }} diff --git a/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml b/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml index 9e640b00df..9f58773174 100644 --- a/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml +++ b/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml @@ -4,13 +4,111 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This template defines a step to run Roslyn Analyzers on the SqlClient build. It uses the -# RoslynAnalyzers@3 task from the Secure Development Team's SDL extension: +# This template runs Roslyn Analyzers (SDL) against a build.proj target using the RoslynAnalyzers@3 +# task from the Secure Development Team's SDL extension, in "Copy Logs Only" mode: # # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-mohanb/security-integration/guardian-wiki/sdl-azdo-extension/roslyn-analyzers-build-task # -# GOTCHA: This step will clobber any existing build output. It should be run _before_ any build -# steps that perform versioning or signing. +# PROVENANCE: Every statement in this file about how the RoslynAnalyzers task behaves is current as +# of task version v3 (RoslynAnalyzers@3, 3.289.0) and was verified against concrete evidence -- the +# actual pipeline run logs, the task definition (task.json / inputMap.json), the gdn-task-lib task +# source, and the Guardian RoslynAnalyzers CLI binaries (Microsoft.Guardian.RoslynAnalyzers*.dll). +# Re-verify these claims if the task's major version changes. +# +# HOW IT WORKS (integrated analyzers + Copy Logs Only): +# .NET [Roslyn] security analyzers are compiler-integrated: they only run as part of the actual +# csc/vbc compilation. build.proj is an orchestrator -- each package's real compile happens in a +# separate "dotnet build .csproj" that build.proj launches with an task. That is +# the crux: anything the RoslynAnalyzers task appends to an *outer* "dotnet build build.proj" +# command (its auto/manual "re-run the build" modes) is an MSBuild global property, and global +# properties do NOT cross an into a child "dotnet build" process. So an injected analyzer +# would only ever see build.proj (which compiles nothing) and never the real projects, producing +# zero results. That is exactly why earlier auto/manual-mode attempts collected 0 SARIF logs. +# +# Instead we use the task's documented "Copy Logs Only" alternative -- integrate the analyzers into +# the build itself, then have the task only collect the results: +# 1. This step runs its own isolated "dotnet build build.proj -t:Build" with +# EnableAnalyzers=true. build.proj forwards that flag into every leaf "dotnet build" it execs +# (via EnableAnalyzersArgument), and src/Directory.Build.props -- which every product project +# imports -- reacts by enabling the full analyzer set and setting ErrorLog to a per-project +# "*.csproj..sarif" log. src/Directory.Build.targets verifies that each leaf compile +# produced its configured log. Because the analyzers and verification are enabled on the leaf +# projects themselves, they run inside the real compiles regardless of the boundary. +# 2. The RoslynAnalyzers@3 task then runs in Copy Logs Only mode (copyLogsOnly: true) and simply +# collects and sanitizes those *.csproj.*.sarif and *.vbproj.*.sarif logs from +# logRootDirectory for SDL/Guardian compliance. It performs no build and no compiler re-run, +# so none of the msBuildVersion / +# msBuildArchitecture / Visual-Studio-setup concerns apply -- the task never needs MSBuild, +# so it is inherently agnostic to the container's VS/MSBuild version (e.g. MSBuild 18 on the +# ltsc2025/vse2026 image). +# +# EnableAnalyzers and IsolatedBuildPath are independent build.proj properties. We set both here: +# EnableAnalyzers turns analysis on; IsolatedBuildPath keeps this analysis build from disturbing +# real build output (see ISOLATION). +# +# WHAT THE TASK ITSELF INJECTS (v3), AND HOW THIS TEMPLATE COVERS IT: +# In its build-driving modes the task appends five MSBuild properties to the compile command and +# injects the analyzers via user-profile ImportBefore/ImportAfter files. This template enables the +# analyzers on the leaf projects instead, reproducing the effects that matter. Item by item: +# +# | Task injection (v3) | Purpose | How this template covers it | +# |----------------------------------------|-------------------------------------|---------------------------------------------------| +# | /p:Features= | Turns on Roslyn IOperation + | latest-recommended: SDK 18 Roslyn has IOperation | +# | "IOperation,flow-analysis" | dataflow so the taint/crypto | on by default, so these rules run. Add | +# | | security rules (CA3xxx/CA5xxx) run. | flow-analysis to | +# | | | Directory.Build.props if any go missing. | +# | /p:CodeAnalysisRuleSet= | Selects the exact SDL rule IDs + | AnalysisLevel=latest-recommended. This is the | +# | ...Sdl.Recommended.Warning.ruleset | severities; disables non-SDL rules. | SDK's own "recommended" mode, NOT the private SDL | +# | | | ruleset, so complete overlap is not guaranteed; | +# | | | see the CAVEAT in Directory.Build.props. The | +# | | | internal IA* rules need the | +# | | | Microsoft.Internal.Analyzers package, which is | +# | | | Microsoft-internal-only and MUST NOT be added to | +# | | | the public governed feed, so they are not | +# | | | reproduced here. See NOTE ON THE INTERNAL IA* | +# | | | RULES below. | +# | /p:TreatWarningsAsErrors=false | Record every diagnostic instead of | false in the | +# | | failing at the first one. | EnableAnalyzers block of Directory.Build.props. | +# | | | Warning-clean compilation is still enforced by | +# | | | the ordinary build later in each job, which does | +# | | | run with TreatWarningsAsErrors=true. | +# | /p:RunCodeAnalysis=false | Disables legacy *binary* FxCop | Already false by default in SDK-style projects; | +# | | (not the Roslyn analyzers). | we never enable it. | +# | /p:GdnRoslynAnalyzersRunId= | Gates the injected props/targets so | Not needed. We enable analyzers directly on the | +# | | they apply only to this build. | leaf projects, so there is no global injection to | +# | | | gate (see the note below). | +# | ImportBefore *.props / ImportAfter | Adds the analyzer assemblies and | Analyzers: EnableNETAnalyzers + | +# | *.targets under %LOCALAPPDATA%\...\ | sets ErrorLog=.sarif. | latest-recommended (no EnforceCodeStyleInBuild). | +# | MSBuild\Current | | ErrorLog: we set | +# | | | $(MSBuildProjectFullPath)....sarif | +# | | | (SARIF v1 -- NO version=2; see the sanitizer | +# | | | note in Directory.Build.props). | +# +# WHY THE TASK'S OWN INJECTION YIELDS 0 SARIF THROUGH build.proj: the ImportAfter *.targets live in +# the user profile, so they ARE imported by build.proj's inner "dotnet build " execs -- but +# they self-gate on $(GdnRoslynAnalyzersRunId), and that property (like CodeAnalysisRuleSet and +# Features) is passed only on the OUTER "dotnet build build.proj" command and does not cross the +# into the child compiles. So the injected targets no-op in the real compiles. Enabling the +# analyzers on the leaf projects (EnableAnalyzers) removes that gate entirely. +# +# NOTE ON THE INTERNAL IA* RULES: +# The SDL-recommended ruleset also contains internal IA* ("Internal Analyzers") rules that ship in +# the Microsoft.Internal.Analyzers package. That package is Microsoft-internal and confidential: it +# is NOT on nuget.org, and it MUST NOT be added to this repo's governed feed +# (sqlclientdrivers.pkgs.visualstudio.com/public/...), which is PUBLIC-scoped -- doing so would +# leak internal tooling and breach its internal-use license. So the leaf-project analysis above +# (AnalysisLevel=latest-recommended) covers the CA* rules but NOT the IA* rules. +# +# The IA* rules can only be run from the PRIVATE ADO.Net project pipelines, where a Microsoft- +# internal NuGet feed is reachable. NuGet.analysis.config adds that feed using an environment- +# variable placeholder and maps Microsoft.Internal.* exclusively to it. Only these analysis builds +# select that config, allowing Microsoft.Internal.Analyzers to be restored and used without +# changing the normal NuGet.config. +# +# ISOLATION: +# This template is self-contained and safe to run at any point in a job -- before or after a real +# build -- because the analysis build writes its binaries to a separate location and never touches +# the real build output, using the IsolatedBuildPath build.proj property. parameters: # Optional arguments to pass to msbuild to indicate what version of dependencies should be used. @@ -31,41 +129,70 @@ parameters: - SqlClient - SqlServer + # The three parameters below mirror build-buildproj-step.yml so the analysis build resolves the + # same package versions as the real build. + # # Version revision translated to build.proj's BuildNumber property at this boundary. - name: revision type: string # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this - # package's version. build.proj recognizes only two such properties: PackageVersionSqlClient - # (shared by the entire SqlClient family: Logging, Abstractions, SqlClient, Azure, and the AKV - # Provider) and PackageVersionSqlServer (Microsoft.SqlServer.Server). Provided by the caller. - # Examples: 'SqlClient' -> -p:PackageVersionSqlClient=7.1.0-preview3 - # 'SqlServer' -> -p:PackageVersionSqlServer=1.0.0 + # package's version. See build-buildproj-step.yml for the full explanation. - name: versionPropertySuffix type: string # Version to stamp on the package. Combined with versionPropertySuffix to form the msbuild # argument, e.g. -p:PackageVersionSqlClient=7.1.0-preview3. - # Always required — compute up-front via the compute-versions stage. - name: packageVersion type: string steps: - # GOTCHA: If there are any blank lines in msbuildCommandLine, it will consider it "multiple - # arguments" and fail. So, don't split msBuildCommandLine into multiple blocks. - - task: securedevelopmentteam.vss-secure-development-tools.build-task-roslynanalyzers.RoslynAnalyzers@3 - displayName: 'Roslyn Analyzers - build.proj Build${{ parameters.packageShortName }}' + # Step 1: Authenticate to the internal Azure Artifacts feed so the leaf restores can pull + # Microsoft.Internal.Analyzers. NuGetAuthenticate sets up the Azure Artifacts credential provider + # for feeds the build identity can access in this organization. + - task: NuGetAuthenticate@1 + displayName: 'Internal analyzers: authenticate internal feed' + + # Step 2: Isolated analysis build. Compiles the package with EnableAnalyzers=true so that every leaf + # "dotnet build" that build.proj execs turns on the full Roslyn analyzer set and verifies its SARIF. + - task: DotNetCoreCLI@2 + displayName: 'Build for Roslyn analysis - build.proj Build${{ parameters.packageShortName }}' inputs: - msBuildArchitecture: x64 - msBuildCommandLine: >- - msbuild - $(REPO_ROOT)/build.proj + command: build + projects: '$(REPO_ROOT)/build.proj' + arguments: >- -t:Build${{ parameters.packageShortName }} -p:Configuration=Release -p:ReferenceType=Package -p:SkipDependencyPack=true - -p:BuildNumber=${{ parameters.revision }} - -p:PackageVersion${{ parameters.versionPropertySuffix }}=${{ parameters.packageVersion }} + -p:BuildNumber="${{ parameters.revision }}" + -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" + -p:IsolatedBuildPath="$(Agent.TempDirectory)/roslyn" + -p:EnableAnalyzers=true + -p:InternalAnalyzers=true + -p:InternalAnalyzersNugetConfig="$(REPO_ROOT)/NuGet.analysis.config" + -p:InternalAnalyzersVersion=$(InternalAnalyzersVersion) ${{ parameters.dependencyArguments }} - msBuildVersion: 17.0 - setupCommandLinePicker: vs2022 + env: + # dotnet restore expands this environment variable into the NuGet feed URL for + # Microsoft.Internal.Analyzers. + INTERNAL_ANALYZERS_FEED: $(InternalAnalyzersFeed) + + # Step 3: List every SARIF file that the collector will ingest. + - pwsh: | + $sarifFiles = @(Get-ChildItem -Path '$(REPO_ROOT)' -Recurse -File -Include '*.csproj.*.sarif', '*.vbproj.*.sarif' | Sort-Object FullName) + Write-Host "Roslyn collector will ingest $($sarifFiles.Count) SARIF file(s):" + $sarifFiles | ForEach-Object { Write-Host " $($_.FullName)" } + displayName: 'List Roslyn SARIF files for collection' + + # Step 4: Collect the analysis results. In Copy Logs Only mode the task does not build or re-run + # the compiler -- it just gathers and sanitizes the *.csproj.*.sarif and *.vbproj.*.sarif logs + # produced by Step 2 and hands them to Guardian/SDL. + - task: securedevelopmentteam.vss-secure-development-tools.build-task-roslynanalyzers.RoslynAnalyzers@3 + displayName: 'Roslyn Analyzers (collect) - build.proj Build${{ parameters.packageShortName }}' + inputs: + copyLogsOnly: true + # Root to search for the *.csproj.*.sarif and *.vbproj.*.sarif logs. The analysis build wrote + # them next to each project under the repo checkout; the collector globs this directory + # recursively. + logRootDirectory: '$(REPO_ROOT)' diff --git a/eng/pipelines/onebranch/variables/onebranch-variables.yml b/eng/pipelines/onebranch/variables/onebranch-variables.yml index fb671961cc..f4ae94431c 100644 --- a/eng/pipelines/onebranch/variables/onebranch-variables.yml +++ b/eng/pipelines/onebranch/variables/onebranch-variables.yml @@ -32,6 +32,15 @@ variables: # SymbolsUploadAccount - group: 'symbols-variables-v3' + # These variables point the SDL Roslyn analysis step at the internal Microsoft.Internal.Analyzers + # package (the "IA*" rules). The package is Microsoft-internal and confidential, so the feed URL + # and pinned version live in this ADO.Net-project variable group rather than in the repo. Consumed + # by eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml. + # + # InternalAnalyzersFeed + # InternalAnalyzersVersion + - group: 'internal-analyzers-variables-v1' + # Well-Known Variables ################################################### # Directory where downloaded pipeline artifacts (NuGet packages from earlier @@ -60,10 +69,12 @@ variables: value: '6.10' # OneBranch supplies a variety of container images we must use for our jobs. + # + # https://eng.ms/docs/products/onebranch/infrastructureandimages/containerimages/containerimages # Windows jobs use this image. - name: WindowsContainerImage - value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + value: onebranch.azurecr.io/windows/ltsc2025/vse2026:latest # Linux jobs use this image. - name: LinuxContainerImage diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 2c4b54b5d0..857bc1b65c 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -32,6 +32,71 @@ + + + + true + true + true + + latest-recommended + + false + + $(MSBuildProjectFullPath).$([System.Guid]::NewGuid().ToString()).sarif + + false + + + + + + + + + + + + + + + diff --git a/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj b/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj index a9474613e6..62a4ef14e0 100644 --- a/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj +++ b/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj @@ -88,12 +88,14 @@ + $(RepoRoot)artifacts/ + - $(RepoRoot)artifacts/$(AssemblyName).notsupported/$(ReferenceType)-$(Configuration)/ + $(ArtifactPath)$(AssemblyName).notsupported/$(ReferenceType)-$(Configuration)/ @@ -105,7 +107,7 @@ --> $(RepoRoot)src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj - $(RepoRoot)artifacts/Microsoft.Data.SqlClient.ref/$(ReferenceType)-$(Configuration)/$(TargetFramework)/Microsoft.Data.SqlClient.dll + $(ArtifactPath)Microsoft.Data.SqlClient.ref/$(ReferenceType)-$(Configuration)/$(TargetFramework)/Microsoft.Data.SqlClient.dll Microsoft.SqlServer.TDS.EndPoint Microsoft.SqlServer.TDS.EndPoint netstandard2.0 - $(OS) diff --git a/tools/PackageCompatibility/NuGet.config b/tools/PackageCompatibility/NuGet.config index 1c814bbc3c..0d35132d82 100644 --- a/tools/PackageCompatibility/NuGet.config +++ b/tools/PackageCompatibility/NuGet.config @@ -10,4 +10,39 @@ --> + + + + + + + + + + + + + + + + + + + + + + + + From 92eb8b46b0e70251b3af0c6af3a19a411521dd64 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:21:38 -0700 Subject: [PATCH 16/51] [Scheduled Run] Localized resource files from OneLocBuild (#4645) Co-authored-by: SqlClient DevOps --- src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx | 7 ++----- .../src/Resources/Strings.pt-BR.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx | 7 ++----- src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx | 7 ++----- .../src/Resources/Strings.zh-Hans.resx | 7 ++----- .../src/Resources/Strings.zh-Hant.resx | 7 ++----- 13 files changed, 26 insertions(+), 65 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx index f85eda7d30..d333b05fd5 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx @@ -2082,11 +2082,8 @@ Pokud je klíčové slovo Integrated Security připojovacího řetězce nastavené na hodnotu true nebo SSPI, nejde nastavit vlastnost AccessTokenCallback. - - Vlastnost AccessToken nebo AccessTokenCallback nelze nastavit, pokud je nastavená vlastnost SspiContextProvider. - - - Vlastnost SspiContextProvider nelze nastavit, pokud je nastavená vlastnost AccessToken nebo AccessTokenCallback. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Pokud je v připojovacím řetězci nastavená možnost Authentication=Active Directory Default, nejde nastavit vlastnost AccessTokenCallback. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx index 7dcd133277..36003faa0f 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx @@ -2082,11 +2082,8 @@ AccessTokenCallback-Eigenschaft kann nicht festgelegt werden, wenn das Schlüsselwort für Verbindungszeichenfolgen 'Integrated Security' auf 'true' oder 'SSPI' gesetzt wurde. - - Die Eigenschaft AccessToken oder AccessTokenCallback kann nicht festgelegt werden, wenn die Eigenschaft SspiContextProvider bereits festgelegt wurde. - - - Die Eigenschaft SspiContextProvider kann nicht festgelegt werden, wenn die Eigenschaft AccessToken oder AccessTokenCallback bereits festgelegt wurde. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Die Eigenschaft AccessTokenCallback kann nicht festgelegt werden, wenn in der Verbindungszeichenfolge "Authentication=Active Directory Default" angegeben wurde. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx index 3471a75894..f24a37eff9 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx @@ -2082,11 +2082,8 @@ No se puede establecer la propiedad AccessTokenCallback si la palabra clave de cadena de conexión "Integrated Security" se ha establecido en "true" o "SSPI". - - No se puede establecer la propiedad AccessToken o AccessTokenCallback si ya se ha establecido la propiedad SspiContextProvider. - - - No se puede establecer la propiedad SspiContextProvider si ya se ha establecido la propiedad AccessToken o AccessTokenCallback. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. No se puede establecer la propiedad AccessTokenCallback si se ha especificado 'Authentication=Active Directory Default' en la cadena de conexión. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx index d73673efbb..32a65bfc0d 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx @@ -2082,11 +2082,8 @@ Impossible de définir la propriété AccessTokenCallback si le mot clé de chaîne de connexion « Sécurité intégrée » a été défini sur « true » ou « SSPI ». - - Nous ne pouvons pas définir la propriété AccessToken ou AccessTokenCallback si la propriété SspiContextProvider a déjà été définie. - - - Nous ne pouvons pas définir la propriété SspiContextProvider si la propriété AccessToken ou AccessTokenCallback a déjà été définie. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Impossible de définir la propriété AccessTokenCallback si 'Authentication=Active Directory Default' a été spécifié dans la chaîne de connexion. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx index 1e2e6764d5..4109e2f808 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx @@ -2082,11 +2082,8 @@ Non è possibile impostare la proprietà AccessTokenCallback se la parola chiave della stringa di connessione 'Integrated Security' è stata impostata su 'true' o 'SSPI'. - - Non è possibile impostare la proprietà AccessToken o AccessTokenCallback se la proprietà SspiContextProvider è già stata impostata. - - - Non è possibile impostare la proprietà SspiContextProvider se la proprietà AccessToken o AccessTokenCallback è già stata impostata. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Impossibile impostare la proprietà AccessTokenCallback se nella stringa di connessione è stato specificato 'Authentication=Active Directory Default'. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx index 6e762be388..b0ff96812b 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx @@ -2082,11 +2082,8 @@ 'Integrated Security' 接続文字列キーワードが 'true' または 'SSPI' に設定されている場合、AccessTokenCallback プロパティを設定できません。 - - SspiContextProvider プロパティが設定されている場合は、AccessToken または AccessTokenCallback プロパティを設定できません。 - - - AccessToken または AccessTokenCallback プロパティが設定されている場合は、SspiContextProvider プロパティを設定できません。 + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. 接続文字列に 'Authentication=Active Directory Default' が指定されている場合、AccessTokenCallback プロパティを設定できません。 diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx index ccca396df1..62433d0ea0 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx @@ -2082,11 +2082,8 @@ 'Integrated Security' 연결 문자열 키워드가 'true' 또는 'SSPI'로 설정된 경우 AccessTokenCallback 속성을 설정할 수 없습니다. - - SspiContextProvider 속성이 설정된 경우 AccessToken 또는 AccessTokenCallback 속성을 설정할 수 없습니다. - - - AccessToken 또는 AccessTokenCallback 속성이 설정된 경우 SspiContextProvider 속성을 설정할 수 없습니다. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. 연결 문자열에 'Authentication=Active Directory Default'가 지정된 경우 AccessTokenCallback 속성을 설정할 수 없습니다. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx index 9a2f4e6f96..c410e654a0 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx @@ -2082,11 +2082,8 @@ Nie można ustawić właściwości AccessTokenCallback, jeśli słowo kluczowe parametrów połączenia „Integrated Security” ma wartość „true” lub „SSPI”. - - Nie można ustawić właściwości AccessToken lub AccessTokenCallback, jeśli ustawiono właściwość SspiContextProvider. - - - Nie można ustawić właściwości SspiContextProvider, jeśli ustawiono właściwość AccessToken lub AccessTokenCallback. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Nie można ustawić właściwości AccessTokenCallback, jeśli w parametrach połączenia określono wartość „Authentication=Active Directory Default”. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx index 2e7593f6fd..31d3ffc029 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx @@ -2082,11 +2082,8 @@ Não é possível definir a propriedade AccessTokenCallback se a palavra-chave da cadeia de conexão 'Integrated Security' tiver sido definida como 'true' ou 'SSPI'. - - Não é possível definir a propriedade AccessToken ou AccessTokenCallback se a propriedade SspiContextProvider já tiver sido definida. - - - Não é possível definir a propriedade SspiContextProvider se a propriedade AccessToken ou AccessTokenCallback já tiver sido definida. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Não é possível definir a propriedade AccessTokenCallback se 'Authentication=Active Directory Default' tiver sido especificado na cadeia de conexão. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx index 17f0d5ba7e..d5d4039980 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx @@ -2082,11 +2082,8 @@ Если ключевому слову строки подключения "Integrated Security" задано значение "true" или "SSPI", свойство "AccessTokenCallback" задать не удастся. - - Если задано свойство SspiContextProvider, свойства AccessToken или AccessTokenCallback настроить невозможно. - - - Если заданы свойства AccessToken или AccessTokenCallback, свойство SspiContextProvider настроить невозможно. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Если в строке подключения указан параметр "Authentication=Active Directory Default", свойство AccessTokenCallback задать не удастся. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx index f874f78401..c62b087f11 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx @@ -2082,11 +2082,8 @@ 'Integrated Security' bağlantı dizesi anahtar sözcüğü 'true' veya 'SSPI' olarak ayarlanmışsa AccessTokenCallback özelliği ayarlanamaz. - - AccessToken veya AccessTokenCallback özelliği, SspiContextProvider özelliği ayarlanmışsa ayarlanamaz. - - - SspiContextProvider özelliği, AccessToken veya AccessTokenCallback özelliği ayarlanmışsa ayarlanamaz. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Bağlantı dizesinde 'Authentication=Active Directory Default' belirtilmişse AccessTokenCallback özelliği ayarlanamaz. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx index 5a5ebe15a6..69ab4523bc 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx @@ -2082,11 +2082,8 @@ 如果 "Integrated Security" 连接字符串关键字设置为 "true" 或 "SSPI",则无法设置 AccessTokenCallback 属性。 - - 如果已设置 SspiContextProvider 属性,则无法设置 AccessToken 或 AccessTokenCallback 属性。 - - - 如果已设置 AccessToken 或 AccessTokenCallback 属性,则无法设置 SspiContextProvider 属性。 + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. 如果在连接字符串中指定了 "Authentication=Active Directory Default",则无法设置 AccessTokenCallback 属性。 diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx index 9b0d8fd673..a5fc02dd7b 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx @@ -2082,11 +2082,8 @@ 如果 'Integrated Security' 連接字串關鍵字已設定為 'true' 或 'SSPI',就不能設定 AccessTokenCallback 屬性。 - - 如果已設定 SspiContextProvider 屬性,就不能設定 AccessToken 或 AccessTokenCallback 屬性。 - - - 如果已設定 AccessToken 或 AccessTokenCallback 屬性,就不能設定 SspiContextProvider 屬性。 + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. 如果已在連接字串中指定 'Authentication=Active Directory Default',就不能設定 AccessTokenCallback 屬性。 From 9eeb13857d0da81f38d675b5a98b3151a83c67d4 Mon Sep 17 00:00:00 2001 From: priyankatiwari08 Date: Mon, 7 Sep 2026 19:54:47 +0530 Subject: [PATCH 17/51] Use official name for US English locale (#4646) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml index ae7dbdd931..6559612464 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml @@ -505,7 +505,7 @@ The following example creates multiple instances of From dc5f8ff39418d14ff833a3d0e577bc0d2a55652e Mon Sep 17 00:00:00 2001 From: priyankatiwari08 Date: Tue, 8 Sep 2026 21:00:49 +0530 Subject: [PATCH 18/51] Resolve PoliCheck severity 2 findings (#4658) Replace inaccessible wording in an internal comment and exclude the Northwind SqlDataAdapter snippet where Country is a required schema identifier. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .config/PolicheckExclusions.xml | 2 +- .../src/Microsoft/Data/SqlClient/SqlCommand.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/PolicheckExclusions.xml b/.config/PolicheckExclusions.xml index a4269513d1..e1129013ba 100644 --- a/.config/PolicheckExclusions.xml +++ b/.config/PolicheckExclusions.xml @@ -1,5 +1,5 @@ SRC/MICROSOFT.DATA.SQLCLIENT/TESTS .YML|.MD|.SQL - NOTICE.TXT|SQLDATAADAPTER.CS + NOTICE.TXT|SQLDATAADAPTER.CS|SQLDATAADAPTER.XML \ No newline at end of file diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs index 294310a08f..c90ac4520d 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs @@ -1306,7 +1306,7 @@ public void ResetCommandTimeout() #region Internal Methods // @TODO: This is only called by SqlCommandBuilder, it should live there. EXCEPT for the one call to ValidateCommand and setting _parameters at the end. Is that really necessary? - // @TODO: This also an crazy long method. + // @TODO: This method is also excessively long. internal void DeriveParameters() { switch (CommandType) From 5a33febcea3ff2ebd0ab53baa080287b57739a35 Mon Sep 17 00:00:00 2001 From: priyankatiwari08 Date: Tue, 8 Sep 2026 23:03:00 +0530 Subject: [PATCH 19/51] Test worldwide text encoding round trips (#4662) Add multilingual Unicode and UTF-8 coverage for parameters, readers, streaming, and bulk copy across sync and async paths. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../BulkCopy/TestBulkCopyWithUTF8.cs | 41 +++- .../GlobalizationEncodingTests.cs | 217 ++++++++++++++++++ 2 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/GlobalizationEncodingTests.cs diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TestBulkCopyWithUTF8.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TestBulkCopyWithUTF8.cs index 21cf670ae7..daa63c7d03 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TestBulkCopyWithUTF8.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TestBulkCopyWithUTF8.cs @@ -4,6 +4,7 @@ using System; using System.Data; +using System.Text; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; using Xunit; @@ -17,11 +18,10 @@ namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy [Trait("Set", "2")] public sealed class TestBulkCopyWithUtf8 : IDisposable { - private static string s_sourceTable = DataTestUtility.GetShortName("SourceTableForUTF8Data"); - private static string s_destinationTable = DataTestUtility.GetShortName("DestinationTableForUTF8Data"); - private static string s_testValue = "test"; - private static byte[] s_testValueInUtf8Bytes = new byte[] { 0x74, 0x65, 0x73, 0x74 }; - private static readonly string s_insertQuery = $"INSERT INTO {s_sourceTable} VALUES('{s_testValue}')"; + private static readonly string s_sourceTable = DataTestUtility.GetShortName("SourceTableForUTF8Data"); + private static readonly string s_destinationTable = DataTestUtility.GetShortName("DestinationTableForUTF8Data"); + private static readonly string s_testValue = GlobalizationTestData.CreatePacketSpanningText(); + private static readonly byte[] s_testValueInUtf8Bytes = Encoding.UTF8.GetBytes(s_testValue); /// /// Constructor: Initializes and populates source and destination tables required for the tests. @@ -37,7 +37,7 @@ public TestBulkCopyWithUtf8() using SqlConnection sourceConnection = new SqlConnection(GetConnectionString(true)); sourceConnection.Open(); - SetupTables(sourceConnection, s_sourceTable, s_destinationTable, s_insertQuery); + SetupTables(sourceConnection, s_sourceTable, s_destinationTable); } /// @@ -61,11 +61,14 @@ public void Dispose() /// /// Builds a connection string with or without Multiple Active Result Sets (MARS) property. /// + /// Whether Multiple Active Result Sets is enabled. + /// A connection string configured with a small packet size for boundary coverage. private string GetConnectionString(bool enableMars) { return new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString) { - MultipleActiveResultSets = enableMars + MultipleActiveResultSets = enableMars, + PacketSize = 512 }.ConnectionString; } @@ -73,14 +76,18 @@ private string GetConnectionString(bool enableMars) /// Creates source and destination tables with a varchar(max) column with a collation setting /// that stores the data in UTF8 encoding and inserts the data in the source table. /// - private void SetupTables(SqlConnection connection, string sourceTable, string destinationTable, string insertQuery) + /// The open connection used to create and populate the tables. + /// The source table name. + /// The destination table name. + private void SetupTables(SqlConnection connection, string sourceTable, string destinationTable) { string columnDefinition = "(str_col varchar(max) COLLATE Latin1_General_100_CS_AS_KS_WS_SC_UTF8)"; DataTestUtility.CreateTable(connection, sourceTable, columnDefinition); DataTestUtility.CreateTable(connection, destinationTable, columnDefinition); using SqlCommand insertCommand = connection.CreateCommand(); - insertCommand.CommandText = insertQuery; - Helpers.TryExecute(insertCommand, insertQuery); + insertCommand.CommandText = $"INSERT INTO {sourceTable} VALUES(@value)"; + insertCommand.Parameters.Add(new SqlParameter("@value", SqlDbType.NVarChar, -1) { Value = s_testValue }); + insertCommand.ExecuteNonQuery(); } /// @@ -105,6 +112,11 @@ public void BulkCopy_Utf8Data_ShouldMatchSource(bool isMarsEnabled, bool enableS using SqlConnection destinationConnection = new SqlConnection(connectionString); destinationConnection.Open(); + using (SqlCommand sourceVerifyCommand = new SqlCommand($"SELECT CAST(str_col AS varbinary(max)) FROM {s_sourceTable}", sourceConnection)) + { + Assert.Equal(s_testValueInUtf8Bytes, sourceVerifyCommand.ExecuteScalar()); + } + // Read data from source table using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection); using SqlDataReader reader = sourceDataCommand.ExecuteReader(CommandBehavior.SequentialAccess); @@ -135,7 +147,7 @@ public void BulkCopy_Utf8Data_ShouldMatchSource(bool isMarsEnabled, bool enableS Assert.Equal(1, Convert.ToInt16(countCommand.ExecuteScalar())); // Read the data from destination table as varbinary to verify the UTF-8 byte sequence - using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection); + using SqlCommand verifyCommand = new SqlCommand($"SELECT CAST(str_col AS varbinary(max)) FROM {s_destinationTable}", destinationConnection); using SqlDataReader verifyReader = verifyCommand.ExecuteReader(CommandBehavior.SequentialAccess); // Verify that we have data in the destination table @@ -170,6 +182,11 @@ public async Task BulkCopy_Utf8Data_ShouldMatchSource_Async(bool isMarsEnabled, using SqlConnection destinationConnection = new SqlConnection(connectionString); await destinationConnection.OpenAsync(); + using (SqlCommand sourceVerifyCommand = new SqlCommand($"SELECT CAST(str_col AS varbinary(max)) FROM {s_sourceTable}", sourceConnection)) + { + Assert.Equal(s_testValueInUtf8Bytes, await sourceVerifyCommand.ExecuteScalarAsync()); + } + // Read data from source table using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection); using SqlDataReader reader = await sourceDataCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); @@ -200,7 +217,7 @@ public async Task BulkCopy_Utf8Data_ShouldMatchSource_Async(bool isMarsEnabled, Assert.Equal(1, Convert.ToInt16(await countCommand.ExecuteScalarAsync())); // Read the data from destination table as varbinary to verify the UTF-8 byte sequence - using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection); + using SqlCommand verifyCommand = new SqlCommand($"SELECT CAST(str_col AS varbinary(max)) FROM {s_destinationTable}", destinationConnection); using SqlDataReader verifyReader = await verifyCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); // Verify that we have data in the destination table diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/GlobalizationEncodingTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/GlobalizationEncodingTests.cs new file mode 100644 index 0000000000..f2ba885b1f --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/GlobalizationEncodingTests.cs @@ -0,0 +1,217 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests +{ + /// + /// Provides representative worldwide text for encoding validation tests. + /// + internal static class GlobalizationTestData + { + internal const string RepresentativeText = + "Latin: Caf\u00E9 / Cafe\u0301 | " + + "Arabic: \u0627\u0644\u0639\u064E\u0631\u064E\u0628\u0650\u064A\u064E\u0651\u0629 | " + + "Devanagari: \u0939\u093F\u0928\u094D\u0926\u0940 | " + + "Thai: \u0E20\u0E32\u0E29\u0E32\u0E44\u0E17\u0E22 | " + + "CJK: \u65E5\u672C\u8A9E \u4E2D\u6587 \uD55C\uAD6D\uC5B4 | " + + "Supplementary: \uD83D\uDE00 \uD834\uDD1E | " + + "Emoji sequence: \uD83D\uDC69\uD83C\uDFFD\u200D\uD83D\uDCBB"; + + /// + /// Repeats the representative text so reads span TDS packets and internal character buffers. + /// + internal static string CreatePacketSpanningText() + { + StringBuilder value = new(); + for (int i = 0; i < 16; i++) + { + value.Append(RepresentativeText); + } + + return value.ToString(); + } + } + + /// + /// Validates exact worldwide text preservation through parameters, SQL Server storage, and reader APIs. + /// + [Trait("Set", "3")] + public static class GlobalizationEncodingTests + { + /// + /// Verifies normal and streamed Unicode parameters round-trip exactly through buffered and sequential + /// readers on synchronous and asynchronous paths. + /// + /// Whether command and reader operations use asynchronous APIs. + /// Whether the input parameter is supplied through a . + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public static async Task UnicodeParameterAndReaderRoundTrip_PreservesWorldwideText(bool useAsync, bool streamInput) + { + string expected = GlobalizationTestData.CreatePacketSpanningText(); + SqlConnectionStringBuilder connectionString = new(DataTestUtility.TCPConnectionString) + { + PacketSize = 512 + }; + + using SqlConnection connection = new(connectionString.ConnectionString); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + + using Table table = new(connection, nameof(UnicodeParameterAndReaderRoundTrip_PreservesWorldwideText), "(Value nvarchar(max) NOT NULL)"); + using StringReader inputReader = new(expected); + using (SqlCommand insert = new($"INSERT INTO {table.Name} (Value) VALUES (@value)", connection)) + { + insert.Parameters.Add(new SqlParameter("@value", SqlDbType.NVarChar, -1) + { + Value = streamInput ? inputReader : expected + }); + + if (useAsync) + { + await insert.ExecuteNonQueryAsync(); + } + else + { + insert.ExecuteNonQuery(); + } + } + + string query = $"SELECT Value FROM {table.Name}"; + string directValue = await ReadDirectValue(connection, query, useAsync); + string streamedValue = await ReadStreamedValue(connection, query, useAsync); + + Assert.Equal(expected, directValue); + Assert.Equal(expected, streamedValue); + Assert.Equal(expected.Length, directValue.Length); + Assert.Equal(expected.Length, streamedValue.Length); + } + + /// + /// Verifies a UTF-8-collated varchar value returns exact worldwide text and the expected UTF-8 bytes + /// on synchronous and asynchronous paths. + /// + /// Whether command and reader operations use asynchronous APIs. + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsUTF8Supported))] + [InlineData(false)] + [InlineData(true)] + public static async Task Utf8VarcharRoundTrip_PreservesWorldwideTextAndBytes(bool useAsync) + { + string expected = GlobalizationTestData.CreatePacketSpanningText(); + SqlConnectionStringBuilder connectionString = new(DataTestUtility.TCPConnectionString) + { + PacketSize = 512 + }; + using SqlConnection connection = new(connectionString.ConnectionString); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + + using Table table = new( + connection, + nameof(Utf8VarcharRoundTrip_PreservesWorldwideTextAndBytes), + "(Value varchar(max) COLLATE Latin1_General_100_CS_AS_KS_WS_SC_UTF8 NOT NULL)"); + using (SqlCommand insert = new($"INSERT INTO {table.Name} (Value) VALUES (@value)", connection)) + { + insert.Parameters.Add(new SqlParameter("@value", SqlDbType.NVarChar, -1) { Value = expected }); + if (useAsync) + { + await insert.ExecuteNonQueryAsync(); + } + else + { + insert.ExecuteNonQuery(); + } + } + + using SqlCommand select = new($"SELECT Value, CONVERT(varbinary(max), Value) FROM {table.Name}", connection); + using SqlDataReader reader = useAsync + ? await select.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : select.ExecuteReader(CommandBehavior.SequentialAccess); + bool hasRow = useAsync ? await reader.ReadAsync() : reader.Read(); + + Assert.True(hasRow); + Assert.Equal(expected, reader.GetString(0)); + Assert.Equal(Encoding.UTF8.GetBytes(expected), reader.GetFieldValue(1)); + Assert.False(useAsync ? await reader.ReadAsync() : reader.Read()); + } + + /// + /// Reads a string through the standard buffered reader path. + /// + /// The open SQL connection used to execute the query. + /// The query that returns one string value. + /// Whether command and reader operations use asynchronous APIs. + /// The string returned by SQL Server. + private static async Task ReadDirectValue(SqlConnection connection, string query, bool useAsync) + { + using SqlCommand command = new(query, connection); + using SqlDataReader reader = useAsync + ? await command.ExecuteReaderAsync() + : command.ExecuteReader(); + bool hasRow = useAsync ? await reader.ReadAsync() : reader.Read(); + + Assert.True(hasRow); + string result = reader.GetString(0); + Assert.False(useAsync ? await reader.ReadAsync() : reader.Read()); + return result; + } + + /// + /// Reads a string through sequential calls with a small buffer so character + /// sequences cross read boundaries. + /// + /// The open SQL connection used to execute the query. + /// The query that returns one string value. + /// Whether command and reader operations use asynchronous APIs. + /// The string returned by SQL Server. + private static async Task ReadStreamedValue(SqlConnection connection, string query, bool useAsync) + { + using SqlCommand command = new(query, connection); + using SqlDataReader reader = useAsync + ? await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : command.ExecuteReader(CommandBehavior.SequentialAccess); + bool hasRow = useAsync ? await reader.ReadAsync() : reader.Read(); + + Assert.True(hasRow); + using TextReader textReader = reader.GetTextReader(0); + char[] buffer = new char[3]; + StringBuilder result = new(); + int charsRead; + do + { + charsRead = useAsync + ? await textReader.ReadAsync(buffer, 0, buffer.Length) + : textReader.Read(buffer, 0, buffer.Length); + result.Append(buffer, 0, charsRead); + } + while (charsRead != 0); + + Assert.False(useAsync ? await reader.ReadAsync() : reader.Read()); + return result.ToString(); + } + } +} From b29753b86ca0ce21fadbecf19ffb51d3a8974245 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:59:56 -0300 Subject: [PATCH 20/51] Pipelines | Use per-package APIScan name/version pairs (#4637) * Pipelines | Use per-package APIScan name/version pairs Each of our NuGet packages is now registered with APIScan under its own name/version pair, so stop attributing every scan to a single global Microsoft.Data.SqlClient / 6.10 registration. - Reinstate the per-job ob_sdl_apiscan_softwareName and ob_sdl_apiscan_versionNumber variables in build-buildproj-job, driven by packageFullName and a new apiScanSoftwareVersion parameter. - Remove softwareName/versionNumber from the globalSdl.apiscan blocks so the build job template is the single place the pair is specified. - Replace ApiScanSoftwareVersion with ApiScanVersionSqlClient (7.1, this branch targets the 7.1.0 release) and ApiScanVersionSqlServer (1.0). - Disable APIScan on validate-signed-package-job, which produces no assemblies and previously relied on the global registration. - Update the SDL section of the OneBranch pipeline design instructions. * Derive APIScan versions from package versions * Log APIScan versions during version computation --- .../onebranch-pipeline-design.instructions.md | 10 ++++-- .../onebranch/jobs/build-buildproj-job.yml | 12 ++++++- .../jobs/validate-signed-package-job.yml | 5 +++ .../onebranch/scripts/compute-versions.ps1 | 33 +++++++++++++++++++ .../scripts/tests/compute-versions.Tests.ps1 | 4 +++ .../onebranch/sqlclient-non-official.yml | 14 +++----- .../onebranch/sqlclient-official.yml | 14 +++----- .../onebranch/stages/build-stages.yml | 16 +++++++++ .../variables/onebranch-variables.yml | 5 --- 9 files changed, 84 insertions(+), 29 deletions(-) diff --git a/.github/instructions/onebranch-pipeline-design.instructions.md b/.github/instructions/onebranch-pipeline-design.instructions.md index 0b647b6be0..272369dfb5 100644 --- a/.github/instructions/onebranch-pipeline-design.instructions.md +++ b/.github/instructions/onebranch-pipeline-design.instructions.md @@ -108,7 +108,8 @@ When `isPreview` is true, pipeline resolves `effective*Version` variables to pre - Variable chain: pipeline YAML → `variables/onebranch-variables.yml` → `variables/common-variables.yml` - All package versions (GA, preview, assembly file) centralized in `variables/common-variables.yml` -- `effective*Version` pipeline variables map to selected version set based on `isPreview` +- The `compute_versions` stage reads canonical versions from MSBuild and publishes effective package, + file-build, and APIScan registration versions for downstream stages - Artifact name variables defined in `variables/onebranch-variables.yml` following `drop__` pattern - `assemblyBuildNumber` derived from first segment of `Build.BuildNumber` only (16-bit limit) - When adding a new package, add GA version, preview version, and assembly file version entries @@ -128,8 +129,11 @@ Variable groups: ## SDL and Compliance - TSA: enabled only in official pipeline; disabled in non-official to avoid spurious alerts -- ApiScan: enabled in both; currently `break: false` pending package registration -- Each build job sets `ob_sdl_apiscan_softwareFolder` to `$(JOB_OUTPUT)/assemblies` and `ob_sdl_apiscan_symbolsFolder` to `$(JOB_OUTPUT)/symbols` +- ApiScan: enabled in both; `break` follows the `breakOnSdlError` parameter +- Each package is registered with APIScan under its own name/version pair, so the `globalSdl.apiscan` blocks deliberately omit `softwareName`/`versionNumber`. `build-buildproj-job.yml` is the single place they are set, via `ob_sdl_apiscan_softwareName` (the package's `packageFullName`) and `ob_sdl_apiscan_versionNumber` (the `apiScanSoftwareVersion` parameter) +- `compute-versions.ps1` derives APIScan registration versions as major.minor from the effective canonical package versions and publishes them as stage outputs. A package name/version pair must still be registered with APIScan before releasing a new major.minor. Consume these as runtime `$(...)` references so values such as `1.0` remain strings rather than being coerced to numbers by template expressions +- Jobs that produce no assemblies (symbol publishing, signed-package validation, version computation) set `ob_sdl_apiscan_enabled: false` rather than reporting a name/version +- Each build job also sets `ob_sdl_apiscan_softwareFolder` and `ob_sdl_apiscan_symbolsFolder` to its per-package `apiScan//dlls` and `apiScan//pdbs` paths - CodeQL, SBOM, Policheck (`break: true`): enabled in both pipelines - asyncSdl `enabled: false` in both; individual sub-tools (CredScan, BinSkim, Armory, Roslyn) configured underneath - Policheck exclusions: `$(REPO_ROOT)\.config\PolicheckExclusions.xml` diff --git a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml index d60ed634c0..f750ed4b91 100644 --- a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml +++ b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml @@ -17,6 +17,12 @@ parameters: - name: apiScanPdbPath type: string + # The APIScan registration version for the package being built. This is the major.minor value + # derived from the canonical package version by the compute_versions stage. The package's + # name/version pair must already be registered with APIScan before the build runs. + - name: apiScanSoftwareVersion + type: string + # True to enable ESRP malware scanning and code signing steps, which should not be run on # non-official pipelines as they access production resources. If true, Signing* parameters must # be provided. @@ -112,9 +118,13 @@ jobs: ob_outputDirectory: '$(JOB_OUTPUT)' - # APIScan per-job configuration for the DLL and PDB folders. + # APIScan per-job configuration. This job template is the single place where the APIScan + # software name and version are set; the pipelines' globalSdl blocks deliberately leave them + # unset so that every scan is attributed to the package it actually covers. ob_sdl_apiscan_softwareFolder: ${{ parameters.apiScanDllPath }} ob_sdl_apiscan_symbolsFolder: ${{ parameters.apiScanPdbPath }} + ob_sdl_apiscan_softwareName: ${{ parameters.packageFullName }} + ob_sdl_apiscan_versionNumber: ${{ parameters.apiScanSoftwareVersion }} steps: - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self diff --git a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml b/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml index afa5aa5918..476bb5de30 100644 --- a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml +++ b/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml @@ -48,6 +48,11 @@ jobs: variables: # More settings at https://aka.ms/obpipelines/yaml/jobs + # This job installs and inspects an already-built package rather than producing assemblies, + # so it has no APIScan software name/version to report. The build jobs scan those assemblies. + - name: ob_sdl_apiscan_enabled + value: false + # Path within the downloaded artifact where NuGet packages are located. - name: artifactPath value: '$(Pipeline.Workspace)\${{ parameters.artifactName }}' diff --git a/eng/pipelines/onebranch/scripts/compute-versions.ps1 b/eng/pipelines/onebranch/scripts/compute-versions.ps1 index ca3eb2c9d9..93e8d318aa 100644 --- a/eng/pipelines/onebranch/scripts/compute-versions.ps1 +++ b/eng/pipelines/onebranch/scripts/compute-versions.ps1 @@ -45,6 +45,8 @@ - VersionRevision - SqlClientPackageVersion - SqlServerPackageVersion + - SqlClientApiScanVersion + - SqlServerApiScanVersion .PARAMETER ProjectPath Absolute or relative path to the repository build.proj file. @@ -273,6 +275,28 @@ function Add-VersionBuildNumber { return $Version } +<# +.SYNOPSIS + Extracts the major.minor components from a package version. + +.PARAMETER Version + Package version beginning with a numeric major.minor pair. + +.OUTPUTS + The major.minor version pair. +#> +function Get-MajorMinorVersion { + param( + [string]$Version + ) + + if ($Version -notmatch "^(\d+)\.(\d+)(?:\.|-|$)") { + throw "Unable to derive a major.minor version from package version '$Version'." + } + + return "$($Matches[1]).$($Matches[2])" +} + <# .SYNOPSIS Emits an Azure DevOps job output variable for consumption by downstream stages. @@ -349,6 +373,15 @@ Write-Host "Effective versions:" Write-Host " SqlClient (family): $sqlClientPackageVersion" Write-Host " SqlServer: $sqlServerPackageVersion" +$sqlClientApiScanVersion = Get-MajorMinorVersion -Version $sqlClientPackageVersion +$sqlServerApiScanVersion = Get-MajorMinorVersion -Version $sqlServerPackageVersion + +Write-Host "APIScan registration versions:" +Write-Host " SqlClient (family): $sqlClientApiScanVersion" +Write-Host " SqlServer: $sqlServerApiScanVersion" + Set-PipelineOutputVariable -Name "SqlClientPackageVersion" -Value $sqlClientPackageVersion Set-PipelineOutputVariable -Name "SqlServerPackageVersion" -Value $sqlServerPackageVersion +Set-PipelineOutputVariable -Name "SqlClientApiScanVersion" -Value $sqlClientApiScanVersion +Set-PipelineOutputVariable -Name "SqlServerApiScanVersion" -Value $sqlServerApiScanVersion Set-PipelineOutputVariable -Name "VersionRevision" -Value $fileVersionBuildNumber diff --git a/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 index ba109c4b71..cd38e6bb66 100644 --- a/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 +++ b/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 @@ -78,6 +78,9 @@ Describe 'compute-versions.ps1 Effective Versions' { $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" $output | Should -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0-preview1\.$script:testBuildNumberPattern" + $output | Should -Match 'SqlClientApiScanVersion;isOutput=true]7\.1' + $output | Should -Match 'SqlServerApiScanVersion;isOutput=true]1\.1' + $output | Should -Match 'APIScan registration versions:\s+SqlClient \(family\): 7\.1\s+SqlServer:\s+1\.1' $output | Should -Match "VersionRevision;isOutput=true]$script:testFileVersionBuildNumber" } @@ -93,6 +96,7 @@ Describe 'compute-versions.ps1 Effective Versions' { $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.42-preview3' $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0' + $output | Should -Match 'SqlServerApiScanVersion;isOutput=true]1\.0' $output | Should -Not -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0\.42' } diff --git a/eng/pipelines/onebranch/sqlclient-non-official.yml b/eng/pipelines/onebranch/sqlclient-non-official.yml index 6316c9654e..3ec9bdbc75 100644 --- a/eng/pipelines/onebranch/sqlclient-non-official.yml +++ b/eng/pipelines/onebranch/sqlclient-non-official.yml @@ -193,16 +193,10 @@ extends: # Use pre-release mode for non-official pipelines. modeType: prerelease - # We have a single "name" registered with APIScan for all of our packages. - # - # https://eng.ms/docs/products/apiscan/onboard/requirements/registersoftware - # - softwareName: Microsoft.Data.SqlClient - - # Similar to the software name, we have a single version registered as well. This has - # nothing to do with the NuGet package version. It is purely an APIScan registration - # value that points to our backend configuration. - versionNumber: $(ApiScanSoftwareVersion) + # The APIScan software name and version are NOT set here. Each package is registered with + # APIScan under its own name/version pair, so every build job sets ob_sdl_apiscan_softwareName + # and ob_sdl_apiscan_versionNumber for the package it builds (see build-buildproj-job.yml). + # Jobs that produce no assemblies disable APIScan instead, via ob_sdl_apiscan_enabled. # We want the standard level of logging. verbosityLevel: standard diff --git a/eng/pipelines/onebranch/sqlclient-official.yml b/eng/pipelines/onebranch/sqlclient-official.yml index 00ec4f3f74..2bc98a655b 100644 --- a/eng/pipelines/onebranch/sqlclient-official.yml +++ b/eng/pipelines/onebranch/sqlclient-official.yml @@ -207,16 +207,10 @@ extends: # Use release mode for official pipelines. modeType: release - # We have a single "name" registered with APIScan for all of our packages. - # - # https://eng.ms/docs/products/apiscan/onboard/requirements/registersoftware - # - softwareName: Microsoft.Data.SqlClient - - # Similar to the software name, we have a single version registered as well. This has - # nothing to do with the NuGet package version. It is purely an APIScan registration - # value that points to our backend configuration. - versionNumber: $(ApiScanSoftwareVersion) + # The APIScan software name and version are NOT set here. Each package is registered with + # APIScan under its own name/version pair, so every build job sets ob_sdl_apiscan_softwareName + # and ob_sdl_apiscan_versionNumber for the package it builds (see build-buildproj-job.yml). + # Jobs that produce no assemblies disable APIScan instead, via ob_sdl_apiscan_enabled. # We want the standard level of logging. verbosityLevel: standard diff --git a/eng/pipelines/onebranch/stages/build-stages.yml b/eng/pipelines/onebranch/stages/build-stages.yml index 49c6616b85..98e656004b 100644 --- a/eng/pipelines/onebranch/stages/build-stages.yml +++ b/eng/pipelines/onebranch/stages/build-stages.yml @@ -86,6 +86,10 @@ stages: value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] + - name: sqlServerApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerApiScanVersion'] ] jobs: # Build Microsoft.Data.SqlClient.Internal.Logging @@ -93,6 +97,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Internal.Logging/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Internal.Logging/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -115,6 +120,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.SqlServer.Server/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.SqlServer.Server/pdbs' + apiScanSoftwareVersion: '$(sqlServerApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -146,6 +152,8 @@ stages: value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] jobs: # Build Microsoft.Data.SqlClient.Extensions.Abstractions @@ -153,6 +161,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Abstractions/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Abstractions/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -189,6 +198,8 @@ stages: value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] jobs: # Build Microsoft.Data.SqlClient @@ -196,6 +207,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -233,6 +245,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Azure/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Azure/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -271,12 +284,15 @@ stages: value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] jobs: - template: /eng/pipelines/onebranch/jobs/build-buildproj-job.yml@self parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' diff --git a/eng/pipelines/onebranch/variables/onebranch-variables.yml b/eng/pipelines/onebranch/variables/onebranch-variables.yml index f4ae94431c..b6d4ce7d5c 100644 --- a/eng/pipelines/onebranch/variables/onebranch-variables.yml +++ b/eng/pipelines/onebranch/variables/onebranch-variables.yml @@ -63,11 +63,6 @@ variables: - name: Packaging.EnableSBOMSigning value: true - # Keep this as a runtime variable reference in globalSdl.apiscan.versionNumber. Passing the - # quoted value directly through an Azure template expression perturbs '6.10' to the number 6.1. - - name: ApiScanSoftwareVersion - value: '6.10' - # OneBranch supplies a variety of container images we must use for our jobs. # # https://eng.ms/docs/products/onebranch/infrastructureandimages/containerimages/containerimages From 7c5daeb228a44114fe7d9608c71826c7bf95da7c Mon Sep 17 00:00:00 2001 From: priyankatiwari08 Date: Wed, 9 Sep 2026 01:25:54 +0530 Subject: [PATCH 21/51] Fix | Scope configurable retry logic assembly resolution to opt-in callers (#4547) * Scope configurable retry logic assembly resolution SqlConfigurableRetryLogicLoader subscribed a handler to AssemblyLoadContext.Default.Resolving in its constructor and never removed it. Because SqlConfigurableRetryLogicManager builds that loader on the default RetryLogicProvider path, simply reading SqlCommand.RetryLogicProvider or SqlConnection.RetryLogicProvider installed a permanent, process-wide assembly resolution hook. The hook then participated in resolving every assembly the host application failed to find, even though the application had not configured any custom retry logic type. It also probed Environment.CurrentDirectory, which is ambient process state unrelated to where the application's binaries live, so assemblies could be resolved from an unintended location. Applications observed this as load failures, and in #2214 as a stack overflow, originating inside SqlClient for assemblies unrelated to SqlClient. Changes: - Probe AppContext.BaseDirectory instead of Environment.CurrentDirectory. - Subscribe the resolving handler only for the duration of the Type.GetType call in LoadType, and remove it in a finally block. - Skip type resolution entirely when no retryLogicType is configured. retryLogicType is optional while retryMethod is required, so configurations selecting a built-in retry method previously still ran the custom type resolution path. Together these mean the handler is never installed unless the application explicitly configured a custom retry logic type, and is gone again as soon as that type has been resolved. Only .NET is affected; the .NET Framework code path does not use AssemblyLoadContext. Refs #2214, #2134 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 * Test | Address review feedback on retry logic assembly probing tests Replaces the internals-based assertions in the configurable retry logic regression tests with a behavioural probe, and restores the UTF-8 BOM that was dropped from the functional test file. The unit test previously read AssemblyLoadContext's private _resolving field to check whether a handler was still subscribed. That reflects into runtime internals we do not own, so the value cannot simply be exposed internally as review suggested. The functional test took a different but also problematic approach, mutating Environment.CurrentDirectory, which is process-wide state and unsafe under parallel test execution. Both now plant a file that is not a valid assembly in the loader's probing directory (AppContext.BaseDirectory) under a name no other component could request, then assert that Assembly.Load reports it as not found. A subscribed handler would locate that file and surface BadImageFormatException instead, so the assertion discriminates cleanly while observing only public behaviour and touching no shared process state. Verified by temporarily reintroducing the unconditional subscription: all four unit tests and the functional test fail, and pass again once removed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 * Test | Cover the successful retry logic type resolution path The existing tests only covered the code paths where the assembly probing handler is never subscribed. The path that legitimately subscribes it, a configured custom retry logic type that actually resolves, was untested, so nothing verified that the handler is removed again afterwards. Add a test that resolves a retry logic factory out of the loader's probing directory and asserts that no probing handler remains subscribed once the loader has been constructed. An invocation counter on the factory confirms the configured type really was resolved and used, rather than the loader silently falling back to the built-in factory. Verified the test is sensitive to both behaviours it covers: pointing the loader's probing directory elsewhere makes it fail, and restoring the unconditional handler subscription makes it fail. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 * Test | Widen probe file cleanup to non-IO failures File.Delete can fail with UnauthorizedAccessException as well as IOException. Catching only the latter meant a cleanup failure could surface as a test failure that had nothing to do with the behaviour under test. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 * Keep retry logic assembly probing active during provider construction Addresses review feedback that scoping the assembly resolving handler to type resolution alone could break existing consumers whose configured retryLogicType has private dependencies. The handler is now subscribed before LoadType and removed only after CreateInstance has run the configured type's constructor and invoked its retry method, so dependency loads triggered during construction are still resolved. Adds Switch.Microsoft.Data.SqlClient.UseLegacyRetryLogicAssemblyResolution as an escape hatch that restores the process-lifetime handler. The switch restores lifetime only; the probing directory remains AppContext.BaseDirectory, so it cannot re-enable the binary planting vector. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 * Treat a whitespace-only retryLogicType as not configured A whitespace value has no type to resolve, so it previously installed the resolving handler, attempted resolution and then fell back to the built-in factory. Skipping the subscription reaches the same provider without changing assembly resolution behavior on the application's behalf. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 * Remove the UseLegacyRetryLogicAssemblyResolution app context switch The switch restored the process-lifetime assembly resolving handler for the one case that scoping cannot cover: a custom retry logic provider whose private dependency is first touched after the provider has been constructed. Shipping a supported way to permanently reinstate a process-wide handler on AssemblyLoadContext.Default works against the point of the change. The driver should not be altering assembly resolution for the whole application on behalf of configurable retry logic, and an affected provider has a simple fix of its own: reference the dependency normally so it lands in deps.json, or register a resolving handler in the application. The handler is now always subscribed only while a configured provider is being resolved and constructed, and only when a custom retry logic type has been configured. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 * Refactor retry assembly resolution subscription Encapsulate the temporary AssemblyLoadContext resolving handler in an IDisposable subscription so cleanup is tied to a using scope. Remove the handler immediately when custom type resolution falls back to the built-in factory, and add direct unit coverage for disposal. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e8018b20-0e43-4de9-93dd-c8080a81a801 --- .../SqlConfigurableRetryLogicLoader.cs | 159 ++++--- .../SqlConfigurableRetryLogicTest.cs | 46 +++ .../SqlConfigurableRetryLogicLoaderTest.cs | 388 ++++++++++++++++++ 3 files changed, 547 insertions(+), 46 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConfigurableRetryLogicLoaderTest.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs index 96eadb9d56..df556a49ba 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs @@ -50,14 +50,6 @@ public SqlConfigurableRetryLogicLoader( string cnnSectionName = SqlConfigurableRetryConnectionSection.Name, string cmdSectionName = SqlConfigurableRetryCommandSection.Name) { - #if NET - // Just only one subscription to this event is required. - // This class isn't supposed to be called more than one time; - // SqlConfigurableRetryLogicManager manages a single instance of this class. - System.Runtime.Loader.AssemblyLoadContext.Default.Resolving -= Default_Resolving; - System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += Default_Resolving; - #endif - AssignProviders(connectionRetryConfigs == null ? null : CreateRetryLogicProvider(cnnSectionName, connectionRetryConfigs), commandRetryConfigs == null ? null : CreateRetryLogicProvider(cmdSectionName, commandRetryConfigs)); } @@ -119,44 +111,70 @@ private static SqlRetryLogicBaseProvider ResolveRetryLogicProvider(string config throw new ArgumentNullException(nameof(retryMethod), StringsHelper.GetString(Strings.SQLCR_RetryMethodNullOrEmpty)); } - Type type = null; - try - { - // Resolve a Type object from the given type name - // Different implementation in .NET Framework & .NET Core - type = LoadType(configurableRetryType); - } - catch (Exception e) - { - // Try to use 'SqlConfigurableRetryFactory' as a default type to discover retry methods - // if there is a problem, resolve using the 'configurableRetryType' type. - type = typeof(SqlConfigurableRetryFactory); - SqlClientEventSource.Log.TryTraceEvent(" Unable to load the '{2}' type; Trying to use the internal `{3}` type: {4}", - TypeName, methodName, configurableRetryType, type.FullName, e); - } + Type type; - // Run the function by using the resolved values to get the SqlRetryLogicBaseProvider object - try + // Whitespace is treated the same as unset: there is no type to resolve, so the + // resolving handler must not be installed for it. + bool customRetryTypeConfigured = !string.IsNullOrWhiteSpace(configurableRetryType); + + // Keep the handler subscribed across both type resolution and provider construction. + // Invoking the configured type's constructor and retry method can load that assembly's + // private dependencies after LoadType has returned. + using (AssemblyResolutionSubscription subscription = new(customRetryTypeConfigured)) { - // Create an instance from the discovered type by its default constructor - object result = CreateInstance(type, retryMethod, option); + if (!customRetryTypeConfigured) + { + // No custom retry logic type was configured, so there is nothing to resolve and + // the built-in factory is used to discover the requested retry method. + type = typeof(SqlConfigurableRetryFactory); + SqlClientEventSource.Log.TryTraceEvent(" No custom retry logic type is configured; Using the internal `{2}` type.", + TypeName, methodName, type.FullName); + } + else + { + try + { + // Resolve a Type object from the given type name + // Different implementation in .NET Framework & .NET Core + type = LoadType(configurableRetryType); + } + catch (Exception e) + { + // The custom type will not be constructed, so the built-in fallback no + // longer needs the custom assembly resolution handler. + subscription.Dispose(); + + // Try to use 'SqlConfigurableRetryFactory' as a default type to discover retry methods + // if there is a problem, resolve using the 'configurableRetryType' type. + type = typeof(SqlConfigurableRetryFactory); + SqlClientEventSource.Log.TryTraceEvent(" Unable to load the '{2}' type; Trying to use the internal `{3}` type: {4}", + TypeName, methodName, configurableRetryType, type.FullName, e); + } + } - if (result is SqlRetryLogicBaseProvider provider) + // Run the function by using the resolved values to get the SqlRetryLogicBaseProvider object + try { - SqlClientEventSource.Log.TryTraceEvent(" The created instace is a {2} type.", - TypeName, methodName, typeof(SqlRetryLogicBaseProvider).FullName); - provider.Retrying += OnRetryingEvent; - return provider; + // Create an instance from the discovered type by its default constructor + object result = CreateInstance(type, retryMethod, option); + + if (result is SqlRetryLogicBaseProvider provider) + { + SqlClientEventSource.Log.TryTraceEvent(" The created instace is a {2} type.", + TypeName, methodName, typeof(SqlRetryLogicBaseProvider).FullName); + provider.Retrying += OnRetryingEvent; + return provider; + } + } + catch (Exception e) + { + // In order to invoke a function dynamically, any type of exception can occur here; + // The main exception and its stack trace will be accessible through the inner exception. + // i.e: Opening a connection or executing a command while invoking a function + // runs the application to the `TargetInvocationException`. + // And using an isolated zone like a specific AppDomain results in an infinite loop. + throw new InvalidOperationException(StringsHelper.GetString(Strings.SQLCR_RetryMethodException, type.FullName, retryMethod), e); } - } - catch (Exception e) - { - // In order to invoke a function dynamically, any type of exception can occur here; - // The main exception and its stack trace will be accessible through the inner exception. - // i.e: Opening a connection or executing a command while invoking a function - // runs the application to the `TargetInvocationException`. - // And using an isolated zone like a specific AppDomain results in an infinite loop. - throw new InvalidOperationException(StringsHelper.GetString(Strings.SQLCR_RetryMethodException, type.FullName, retryMethod), e); } SqlClientEventSource.Log.TryTraceEvent(" Unable to resolve a valid provider; Returns `null`.", TypeName, methodName); @@ -334,13 +352,53 @@ private static ICollection SplitErrorNumberList(string list) } #region Type Resolution - + + internal sealed class AssemblyResolutionSubscription : IDisposable + { + #if NET + private bool _isSubscribed; + #endif + + internal AssemblyResolutionSubscription(bool subscribe) + { + #if NET + if (subscribe) + { + AssemblyLoadContext.Default.Resolving += Default_Resolving; + _isSubscribed = true; + } + #endif + } + + public void Dispose() + { + #if NET + if (_isSubscribed) + { + AssemblyLoadContext.Default.Resolving -= Default_Resolving; + _isSubscribed = false; + } + #endif + } + } + #if NET + /// + /// The directory that user-supplied configurable retry logic assemblies are probed from. + /// + /// + /// This is deliberately the application base directory rather than the current working + /// directory. The working directory is ambient process state that can be changed at any + /// time and is not necessarily related to where the application's binaries live, so + /// probing it can load assemblies from an unintended and untrusted location. + /// + private static string ProbingDirectory => AppContext.BaseDirectory; + private static Assembly AssemblyResolver(AssemblyName arg) { string methodName = nameof(AssemblyResolver); - string fullPath = MakeFullPath(Environment.CurrentDirectory, arg.Name); + string fullPath = MakeFullPath(ProbingDirectory, arg.Name); SqlClientEventSource.Log.TryTraceEvent(" Looking for '{2}' assembly by '{3}' full path." , TypeName, methodName, arg, fullPath); @@ -348,13 +406,19 @@ private static Assembly AssemblyResolver(AssemblyName arg) } /// - /// Load assemblies on request. + /// Load dependencies of a user-supplied configurable retry logic assembly on request. /// + /// + /// This handler is only subscribed while a configured retry logic provider is being + /// resolved and constructed, and only when a custom retry logic type has been configured. + /// It must not remain subscribed to after that, + /// because doing so changes assembly resolution behavior for the entire application. + /// private static Assembly Default_Resolving(AssemblyLoadContext arg1, AssemblyName arg2) { string methodName = nameof(Default_Resolving); - string target = MakeFullPath(Environment.CurrentDirectory, arg2.Name); + string target = MakeFullPath(ProbingDirectory, arg2.Name); SqlClientEventSource.Log.TryTraceEvent(" Looking for '{2}' assembly that is requested by '{3}' ALC from '{4}' path." , TypeName, methodName, arg2, arg1, target); @@ -371,7 +435,10 @@ private static Type LoadType(string fullyQualifiedName) string methodName = nameof(LoadType); SqlClientEventSource.Log.TryTraceEvent(" Entry point.", TypeName, methodName); - var result = Type.GetType(fullyQualifiedName, AssemblyResolver, TypeResolver); + Type result; + + result = Type.GetType(fullyQualifiedName, AssemblyResolver, TypeResolver); + if (result != null) { SqlClientEventSource.Log.TryTraceEvent(" The '{2}' type is resolved.", diff --git a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConfigurableRetryLogicTest.cs b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConfigurableRetryLogicTest.cs index b04115b191..b24a32e804 100644 --- a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConfigurableRetryLogicTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConfigurableRetryLogicTest.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using System.IO; +using System.Reflection; using System.Threading.Tasks; using Xunit; @@ -80,5 +82,49 @@ public void ValidateRetryParameters() option.AuthorizedSqlCondition = null; SqlConfigurableRetryFactory.CreateIncrementalRetryProvider(option); } + +#if NET + /// + /// Regression test: triggering the configurable retry logic loader through its normal + /// entry points must not leave a process-wide + /// resolving handler + /// installed. Such a handler participates in resolution of every assembly the host + /// application fails to find, and serves them out of this component's probing directory, + /// which can load code from an unintended location. + /// + [Fact] + public void RetryLogicProviderDoesNotLeaveAssemblyProbingEnabled() + { + // Touch the default retry logic providers to force SqlConfigurableRetryLogicLoader + // construction via its normal code path. + Assert.NotNull(new SqlCommand().RetryLogicProvider); + Assert.NotNull(new SqlConnection().RetryLogicProvider); + + // A file that is not a valid assembly, planted in the loader's probing directory + // under a name no other component could be asking for. If a handler is still + // subscribed it finds this file and fails with BadImageFormatException. With correct + // behavior the runtime never looks here and reports the assembly as simply not found. + string assemblySimpleName = "MdsProbeAssembly_" + Guid.NewGuid().ToString("N"); + string plantedFile = Path.Combine(AppContext.BaseDirectory, assemblySimpleName + ".dll"); + + File.WriteAllText(plantedFile, "not an assembly"); + try + { + Assert.Throws( + () => Assembly.Load(new AssemblyName(assemblySimpleName))); + } + finally + { + try + { + File.Delete(plantedFile); + } + catch (IOException) + { + // Best effort cleanup. + } + } + } +#endif } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConfigurableRetryLogicLoaderTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConfigurableRetryLogicLoaderTest.cs new file mode 100644 index 0000000000..d4ef506991 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConfigurableRetryLogicLoaderTest.cs @@ -0,0 +1,388 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#if NET + +using System; +using System.IO; +using System.Reflection; +using System.Runtime.Loader; +using Microsoft.Data.SqlClient.Tests.Common; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests; + +/// +/// Unit tests validating that never leaves a +/// process-wide assembly resolving handler attached to +/// . +/// +/// +/// A handler left attached there participates in resolution of every assembly the host +/// application fails to find, not just the retry logic assembly the loader was interested in. +/// That silently changes assembly loading behaviour for code that never opted in to this +/// feature, and can serve unrelated assemblies out of this component's probing directory. +/// +[Collection(AppContextSwitchTestCollection.Name)] +public class SqlConfigurableRetryLogicLoaderTest +{ + /// + /// The default code path: no configuration at all. The loader must not subscribe to the + /// default load context. + /// + [Fact] + public void Constructor_WithNoConfiguration_DoesNotLeaveAssemblyProbingEnabled() + { + _ = new SqlConfigurableRetryLogicLoader(null, null); + + AssertNoAssemblyProbingHandlerInstalled(); + } + + /// + /// A configuration that supplies only a retry method, which is the documented way to select + /// one of the built-in retry providers. No custom type is being requested, so no assembly + /// resolution is required and no handler may be subscribed - not even transiently. + /// + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Constructor_WithoutRetryLogicType_DoesNotLeaveAssemblyProbingEnabled(string? retryLogicType) + { + TestRetryConnectionSection section = CreateSection(retryLogicType); + + SqlConfigurableRetryLogicLoader loader = new(section, null); + + // The built-in factory still resolves the requested method. + Assert.NotNull(loader.ConnectionProvider); + AssertNoAssemblyProbingHandlerInstalled(); + } + + /// + /// A configuration that requests a custom retry logic type legitimately needs assembly + /// resolution, but the handler must be removed again once type resolution has finished. + /// + [Fact] + public void Constructor_WithUnresolvableRetryLogicType_DoesNotLeaveAssemblyProbingEnabled() + { + TestRetryConnectionSection section = + CreateSection("Some.Namespace.NoSuchType, Some.Assembly.That.Does.Not.Exist"); + + SqlConfigurableRetryLogicLoader loader = new(section, null); + + // Resolution fails and falls back to the built-in factory rather than throwing. + Assert.NotNull(loader.ConnectionProvider); + AssertNoAssemblyProbingHandlerInstalled(); + } + + /// + /// The path a custom retry logic type is actually resolved on. This is the only path that + /// legitimately installs the probing handler, and it must be removed again once resolution + /// has finished. + /// + /// + /// The retry logic type is resolved out of this test assembly, which is reached through the + /// loader's probing directory rather than through normal assembly resolution. The invocation + /// counter confirms the configured type really was resolved and used, so this is exercising + /// the successful branch of type resolution rather than silently falling back to the built-in + /// factory. + /// + [Fact] + public void Constructor_WithResolvableRetryLogicType_DoesNotLeaveAssemblyProbingEnabled() + { + RunWithProbedRetryLogicFactory(loader => + { + Assert.NotNull(loader.ConnectionProvider); + AssertNoAssemblyProbingHandlerInstalled(); + }); + } + + /// + /// The probing handler must stay subscribed until the configured provider has been fully + /// constructed, not just until the type has been resolved. + /// + /// + /// Instantiating the configured type and invoking its retry method can trigger loads of that + /// assembly's private dependencies, and those loads happen after type resolution has already + /// returned. Unsubscribing too early would break providers that depend on that behaviour. The + /// factory method records the state of the handler at the moment it runs, which is inside the + /// window that has to remain open. + /// + [Fact] + public void RetryLogicTypeResolution_KeepsAssemblyProbingEnabledWhileProviderIsConstructed() + { + string probeAssemblyName = NewProbeAssemblyName(); + string plantedFile = PlantProbeFile(probeAssemblyName); + + ProbedRetryLogicFactory.ProbeAssemblyName = probeAssemblyName; + ProbedRetryLogicFactory.ProbingHandlerInstalledDuringInvocation = null; + + try + { + RunWithProbedRetryLogicFactory(loader => + { + Assert.NotNull(loader.ConnectionProvider); + + Assert.True( + ProbedRetryLogicFactory.ProbingHandlerInstalledDuringInvocation, + "The assembly probing handler was not subscribed while the configured retry " + + "logic provider was being constructed."); + }); + + // ...and it must be gone again afterwards. + AssertNoAssemblyProbingHandlerInstalled(); + } + finally + { + ProbedRetryLogicFactory.ProbeAssemblyName = null; + ProbedRetryLogicFactory.ProbingHandlerInstalledDuringInvocation = null; + DeleteProbeFile(plantedFile); + } + } + + /// + /// Disposing an + /// removes its handler from the default assembly load context. + /// + [Fact] + public void AssemblyResolutionSubscription_DisposeRemovesAssemblyProbingHandler() + { + string subscribedProbeName = NewProbeAssemblyName(); + string subscribedProbeFile = PlantProbeFile(subscribedProbeName); + string disposedProbeName = NewProbeAssemblyName(); + string disposedProbeFile = PlantProbeFile(disposedProbeName); + + try + { + using SqlConfigurableRetryLogicLoader.AssemblyResolutionSubscription subscription = + new(subscribe: true); + + Assert.True(IsProbingHandlerInstalled(subscribedProbeName)); + + subscription.Dispose(); + + Assert.False(IsProbingHandlerInstalled(disposedProbeName)); + } + finally + { + DeleteProbeFile(subscribedProbeFile); + DeleteProbeFile(disposedProbeFile); + } + } + + /// + /// Builds a configuration that resolves out of this test + /// assembly through the loader's probing directory, constructs a loader from it, and hands the + /// loader to . + /// + private static void RunWithProbedRetryLogicFactory(Action assert) + { + Assembly testAssembly = typeof(SqlConfigurableRetryLogicLoaderTest).Assembly; + string assemblySimpleName = testAssembly.GetName().Name!; + + // The loader probes for '.dll'. The test assembly's file name does not + // necessarily match its simple name, so make a copy that does. + string probePath = Path.Combine(AppContext.BaseDirectory, assemblySimpleName + ".dll"); + bool copied = false; + if (!File.Exists(probePath)) + { + File.Copy(testAssembly.Location, probePath); + copied = true; + } + + try + { + TestRetryConnectionSection section = CreateSection( + $"{typeof(ProbedRetryLogicFactory).FullName}, {assemblySimpleName}"); + section.RetryMethod = nameof(ProbedRetryLogicFactory.CreateProbedRetryProvider); + + ProbedRetryLogicFactory.InvocationCount = 0; + + SqlConfigurableRetryLogicLoader loader = new(section, null); + + Assert.Equal(1, ProbedRetryLogicFactory.InvocationCount); + + assert(loader); + } + finally + { + if (copied) + { + DeleteProbeFile(probePath); + } + } + } + + private static TestRetryConnectionSection CreateSection(string? retryLogicType) => + new() + { + RetryLogicType = retryLogicType!, + RetryMethod = nameof(SqlConfigurableRetryFactory.CreateFixedRetryProvider), + NumberOfTries = 2, + DeltaTime = TimeSpan.FromSeconds(1), + MinTimeInterval = TimeSpan.Zero, + MaxTimeInterval = TimeSpan.FromSeconds(10), + }; + + /// + /// Asserts that a failed assembly load is not served out of the loader's probing directory, + /// which can only happen while a resolving handler installed by + /// is subscribed to + /// . + /// + /// + /// A file that is not a valid assembly is planted in the probing directory under a name no + /// other component could be asking for. If a handler is still subscribed it finds that file + /// and tries to load it, which surfaces as . With no + /// handler subscribed the runtime never looks there and reports the assembly as simply not + /// found. This asserts the behaviour that actually matters to a host application rather than + /// inspecting loader or runtime internals. + /// + private static void AssertNoAssemblyProbingHandlerInstalled() + { + string assemblySimpleName = NewProbeAssemblyName(); + string plantedFile = PlantProbeFile(assemblySimpleName); + + try + { + Assert.False( + IsProbingHandlerInstalled(assemblySimpleName), + "A resolving handler that probes the loader's probing directory is subscribed to " + + "the default assembly load context."); + } + finally + { + DeleteProbeFile(plantedFile); + } + } + + /// + /// Reports whether a resolving handler that serves assemblies out of the loader's probing + /// directory is currently subscribed to . + /// + /// + /// This distinguishes the two states using only public behaviour, which is what actually + /// matters to a host application. With such a handler subscribed the planted file is found + /// and an attempt is made to load it, which fails as + /// because it is not a valid assembly. With no such + /// handler subscribed the runtime never looks in that directory and reports the assembly as + /// simply not found. + /// + internal static bool IsProbingHandlerInstalled(string assemblySimpleName) + { + try + { + Assembly.Load(new AssemblyName(assemblySimpleName)); + + // Unreachable: the planted file is deliberately not a valid assembly, so a handler + // that found it cannot have loaded it successfully. + return true; + } + catch (BadImageFormatException) + { + return true; + } + catch (FileNotFoundException) + { + return false; + } + } + + private static string NewProbeAssemblyName() => + "MdsProbeAssembly_" + Guid.NewGuid().ToString("N"); + + /// + /// Plants a file that is not a valid assembly in the loader's probing directory, under a + /// name no other component could be asking for. + /// + private static string PlantProbeFile(string assemblySimpleName) + { + // The probing directory is the application base directory, which for a test run is the + // directory the test assembly was loaded from. + string plantedFile = Path.Combine(AppContext.BaseDirectory, assemblySimpleName + ".dll"); + + File.WriteAllText(plantedFile, "not an assembly"); + + return plantedFile; + } + + /// + /// Removes a planted probe file. Cleanup failures are ignored so that a test reports on + /// product behaviour rather than on the state of the file system. + /// + private static void DeleteProbeFile(string plantedFile) + { + try + { + File.Delete(plantedFile); + } + catch (IOException) + { + // The file is in use, most likely because it was successfully loaded as an assembly + // and is therefore locked for the lifetime of the process. + } + catch (UnauthorizedAccessException) + { + // The file is read only, or the caller lacks permission to delete it. + } + } + + private sealed class TestRetryConnectionSection : ISqlConfigurableRetryConnectionSection + { + public TimeSpan DeltaTime { get; set; } + + public TimeSpan MaxTimeInterval { get; set; } + + public TimeSpan MinTimeInterval { get; set; } + + public int NumberOfTries { get; set; } + + public string RetryLogicType { get; set; } = string.Empty; + + public string RetryMethod { get; set; } = string.Empty; + + public string TransientErrors { get; set; } = string.Empty; + } +} + +/// +/// A retry logic factory that is resolved through the loader's probing directory rather than +/// through normal assembly resolution, so tests can tell a successful custom type resolution +/// apart from a silent fallback to the built-in factory. +/// +/// +/// This has to be a public, non-nested type because the loader discovers candidates by +/// enumerating the resolved assembly's exported types. +/// +public static class ProbedRetryLogicFactory +{ + internal static int InvocationCount; + + /// + /// When set to the simple name of a planted probe assembly, the factory method records whether + /// the loader's probing handler was subscribed at the moment it ran. + /// + internal static string? ProbeAssemblyName; + + /// + /// The state of the loader's probing handler at the moment the factory method last ran, or null + /// if it was not recorded. + /// + internal static bool? ProbingHandlerInstalledDuringInvocation; + + public static SqlRetryLogicBaseProvider CreateProbedRetryProvider(SqlRetryLogicOption option) + { + InvocationCount++; + + if (ProbeAssemblyName is not null) + { + ProbingHandlerInstalledDuringInvocation = + SqlConfigurableRetryLogicLoaderTest.IsProbingHandlerInstalled(ProbeAssemblyName); + } + + return SqlConfigurableRetryFactory.CreateFixedRetryProvider(option); + } +} + +#endif From 246ec7bd85d23b5bb4f0d35ea2cc3ea9e5163c22 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:06:20 -0700 Subject: [PATCH 22/51] Pipelines | Move CI and PR jobs off Microsoft-hosted agents (#4515) * Move CI and PR pipelines to SQL Server 2025 agent images Retire every SQL Server 2022 agent image in favour of the SQL Server 2025 equivalents: ADO-MMS22-SQL22 -> ADO-MMS25-SQL25 ADO-UB22-SQL22 -> ADO-UB24-SQL25 This covers the CI test configurations, the PR pipeline platform list, the Azure package integration test jobs (including the SQL root path, which becomes SQL25RootPath), the Managed Instance jobs, the stress test jobs, and the Linux enclave configuration. Two notes on the change: - CI test stage names derive from the image keys, so stages such as Win22_Sql22 are now named Win25_Sql25. Any branch policies or required status checks that reference the old stage names will need to be updated. - The Linux SQL configuration previously ran on both ADO-UB20-SQL22 and ADO-UB22-SQL22. We only have a single Ubuntu SQL 2025 image, so it now runs on ADO-UB24-SQL25 alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Clarify why the Linux enclave stage key is left unchanged The Linux enclave image key doubles as the generated ADO stage name and is referenced by branch policies and required status checks, so it is kept as-is. The 'Sql19' suffix also remains accurate because these tests target a remote Enclave-enabled SQL Server 2019; only the agent image moved to Ubuntu 24. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Install sqlcmd on Linux agents before configuring SQL Server The SQL Server 2025 agent images ship the engine but not the command line tools, so the Linux SQL configuration steps failed with: ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations. Add a shared 'Install sqlcmd [Linux]' step that reuses an existing sqlcmd when the image provides one and otherwise installs mssql-tools18, then publishes the resolved path via the SqlCmdBin variable. Both the PR and CI Linux configuration steps now run it, and the CI step no longer hardcodes /opt/mssql-tools/bin/sqlcmd, which does not exist on these images. sqlcmd from mssql-tools18 encrypts by default, so the step also publishes SqlCmdTrustArg ('-C') to trust the local server's self-signed certificate. Without it every connection to localhost would fail certificate validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Add .NET 10 test coverage to the CI-SqlClient pipeline Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Move CI and PR jobs off Microsoft-hosted agents All jobs that previously ran on the Microsoft-hosted 'Azure Pipelines' pool now run on our 1ES pools, with the sole exception of macOS jobs, since our 1ES pools do not offer macOS images. - Parameterize the pool name and image for the secrets, pack, verify-nuget, and code-coverage jobs in both the CI and PR pipelines, defaulting to the 1ES pool with a Linux image (or a Windows image, for verify-nuget). - Point the Abstractions package Linux and Windows test jobs at ADO-UB24-SQL25 and ADO-MMS25-SQL25. - Drop the redundant hosted Linux and Windows Azure package test jobs. The self-hosted integration jobs already cover the same runtimes, and additionally exercise a local SQL Server. - Pass explicit pool names from the stress and Kerberos pipelines, which reuse these templates but do not import the CI build variables. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Move the CI package pipeline to the SQL Server 2025 agent images The nightly package build pipeline still offered ADO-UB24 and ADO-Win25 as its agentImage choices. Move it onto the same images the rest of CI now uses: ADO-UB24-SQL25 and ADO-MMS25-SQL25. The job only runs build.proj Pack, so it does not depend on the SQL Server instance these images carry. Aligning them means we maintain one set of agent images rather than two. Note that both ADO-1ES-Pool and ADO-CI-1ES-Pool must publish these images before this merges, since the pool is chosen based on whether the build is internal or public. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Move the GitHub sync pipeline off the hosted ubuntu-latest agent This was the last job outside of macOS still running on a Microsoft-hosted agent. Move it to ADO-UB24-SQL25 on the 1ES pool, selecting the pool based on the project the way the stress and package pipelines do. The job runs a PowerShell script with pwsh, which the ADO-UB24-SQL25 image already provides -- the stress job relies on the same thing without installing PowerShell first. After this change, the only remaining Microsoft-hosted jobs are the macOS ones, since our 1ES pools do not offer macOS images. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Support both 1ES and hosted pools in parameterized job templates The pack, secrets, coverage and NuGet verification templates now take poolName/vmImage parameters, but selected the image with a 1ES imageOverride demand unconditionally. That breaks if a caller (or a variable group that has not been migrated yet) still points poolName at the hosted 'Azure Pipelines' pool, which requires vmImage instead. Use the same conditional pool block the test-* job templates already use, so these templates work with either pool type. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Move CI and PR pipelines to SQL Server 2025 agent images Retire every SQL Server 2022 agent image in favour of the SQL Server 2025 equivalents: ADO-MMS22-SQL22 -> ADO-MMS25-SQL25 ADO-UB22-SQL22 -> ADO-UB24-SQL25 This covers the CI test configurations, the PR pipeline platform list, the Azure package integration test jobs (including the SQL root path, which becomes SQL25RootPath), the Managed Instance jobs, the stress test jobs, and the Linux enclave configuration. Two notes on the change: - CI test stage names derive from the image keys, so stages such as Win22_Sql22 are now named Win25_Sql25. Any branch policies or required status checks that reference the old stage names will need to be updated. - The Linux SQL configuration previously ran on both ADO-UB20-SQL22 and ADO-UB22-SQL22. We only have a single Ubuntu SQL 2025 image, so it now runs on ADO-UB24-SQL25 alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Clarify why the Linux enclave stage key is left unchanged The Linux enclave image key doubles as the generated ADO stage name and is referenced by branch policies and required status checks, so it is kept as-is. The 'Sql19' suffix also remains accurate because these tests target a remote Enclave-enabled SQL Server 2019; only the agent image moved to Ubuntu 24. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Install sqlcmd on Linux agents before configuring SQL Server The SQL Server 2025 agent images ship the engine but not the command line tools, so the Linux SQL configuration steps failed with: ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations. Add a shared 'Install sqlcmd [Linux]' step that reuses an existing sqlcmd when the image provides one and otherwise installs mssql-tools18, then publishes the resolved path via the SqlCmdBin variable. Both the PR and CI Linux configuration steps now run it, and the CI step no longer hardcodes /opt/mssql-tools/bin/sqlcmd, which does not exist on these images. sqlcmd from mssql-tools18 encrypts by default, so the step also publishes SqlCmdTrustArg ('-C') to trust the local server's self-signed certificate. Without it every connection to localhost would fail certificate validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Add .NET 10 test coverage to the CI-SqlClient pipeline Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Move CI and PR pipelines to SQL Server 2025 agent images Retire every SQL Server 2022 agent image in favour of the SQL Server 2025 equivalents: ADO-MMS22-SQL22 -> ADO-MMS25-SQL25 ADO-UB22-SQL22 -> ADO-UB24-SQL25 This covers the CI test configurations, the PR pipeline platform list, the Azure package integration test jobs (including the SQL root path, which becomes SQL25RootPath), the Managed Instance jobs, the stress test jobs, and the Linux enclave configuration. Two notes on the change: - CI test stage names derive from the image keys, so stages such as Win22_Sql22 are now named Win25_Sql25. Any branch policies or required status checks that reference the old stage names will need to be updated. - The Linux SQL configuration previously ran on both ADO-UB20-SQL22 and ADO-UB22-SQL22. We only have a single Ubuntu SQL 2025 image, so it now runs on ADO-UB24-SQL25 alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Clarify why the Linux enclave stage key is left unchanged The Linux enclave image key doubles as the generated ADO stage name and is referenced by branch policies and required status checks, so it is kept as-is. The 'Sql19' suffix also remains accurate because these tests target a remote Enclave-enabled SQL Server 2019; only the agent image moved to Ubuntu 24. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Install sqlcmd on Linux agents before configuring SQL Server The SQL Server 2025 agent images ship the engine but not the command line tools, so the Linux SQL configuration steps failed with: ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations. Add a shared 'Install sqlcmd [Linux]' step that reuses an existing sqlcmd when the image provides one and otherwise installs mssql-tools18, then publishes the resolved path via the SqlCmdBin variable. Both the PR and CI Linux configuration steps now run it, and the CI step no longer hardcodes /opt/mssql-tools/bin/sqlcmd, which does not exist on these images. sqlcmd from mssql-tools18 encrypts by default, so the step also publishes SqlCmdTrustArg ('-C') to trust the local server's self-signed certificate. Without it every connection to localhost would fail certificate validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Add .NET 10 test coverage to the CI-SqlClient pipeline Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Move agent configuration to pipeline roots Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Move CI and PR pipelines to SQL Server 2025 agent images Retire every SQL Server 2022 agent image in favour of the SQL Server 2025 equivalents: ADO-MMS22-SQL22 -> ADO-MMS25-SQL25 ADO-UB22-SQL22 -> ADO-UB24-SQL25 This covers the CI test configurations, the PR pipeline platform list, the Azure package integration test jobs (including the SQL root path, which becomes SQL25RootPath), the Managed Instance jobs, the stress test jobs, and the Linux enclave configuration. Two notes on the change: - CI test stage names derive from the image keys, so stages such as Win22_Sql22 are now named Win25_Sql25. Any branch policies or required status checks that reference the old stage names will need to be updated. - The Linux SQL configuration previously ran on both ADO-UB20-SQL22 and ADO-UB22-SQL22. We only have a single Ubuntu SQL 2025 image, so it now runs on ADO-UB24-SQL25 alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Clarify why the Linux enclave stage key is left unchanged The Linux enclave image key doubles as the generated ADO stage name and is referenced by branch policies and required status checks, so it is kept as-is. The 'Sql19' suffix also remains accurate because these tests target a remote Enclave-enabled SQL Server 2019; only the agent image moved to Ubuntu 24. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Install sqlcmd on Linux agents before configuring SQL Server The SQL Server 2025 agent images ship the engine but not the command line tools, so the Linux SQL configuration steps failed with: ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations. Add a shared 'Install sqlcmd [Linux]' step that reuses an existing sqlcmd when the image provides one and otherwise installs mssql-tools18, then publishes the resolved path via the SqlCmdBin variable. Both the PR and CI Linux configuration steps now run it, and the CI step no longer hardcodes /opt/mssql-tools/bin/sqlcmd, which does not exist on these images. sqlcmd from mssql-tools18 encrypts by default, so the step also publishes SqlCmdTrustArg ('-C') to trust the local server's self-signed certificate. Without it every connection to localhost would fail certificate validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Add .NET 10 test coverage to the CI-SqlClient pipeline Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Move CI and PR pipelines to SQL Server 2025 agent images Retire every SQL Server 2022 agent image in favour of the SQL Server 2025 equivalents: ADO-MMS22-SQL22 -> ADO-MMS25-SQL25 ADO-UB22-SQL22 -> ADO-UB24-SQL25 This covers the CI test configurations, the PR pipeline platform list, the Azure package integration test jobs (including the SQL root path, which becomes SQL25RootPath), the Managed Instance jobs, the stress test jobs, and the Linux enclave configuration. Two notes on the change: - CI test stage names derive from the image keys, so stages such as Win22_Sql22 are now named Win25_Sql25. Any branch policies or required status checks that reference the old stage names will need to be updated. - The Linux SQL configuration previously ran on both ADO-UB20-SQL22 and ADO-UB22-SQL22. We only have a single Ubuntu SQL 2025 image, so it now runs on ADO-UB24-SQL25 alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Clarify why the Linux enclave stage key is left unchanged The Linux enclave image key doubles as the generated ADO stage name and is referenced by branch policies and required status checks, so it is kept as-is. The 'Sql19' suffix also remains accurate because these tests target a remote Enclave-enabled SQL Server 2019; only the agent image moved to Ubuntu 24. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Install sqlcmd on Linux agents before configuring SQL Server The SQL Server 2025 agent images ship the engine but not the command line tools, so the Linux SQL configuration steps failed with: ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations. Add a shared 'Install sqlcmd [Linux]' step that reuses an existing sqlcmd when the image provides one and otherwise installs mssql-tools18, then publishes the resolved path via the SqlCmdBin variable. Both the PR and CI Linux configuration steps now run it, and the CI step no longer hardcodes /opt/mssql-tools/bin/sqlcmd, which does not exist on these images. sqlcmd from mssql-tools18 encrypts by default, so the step also publishes SqlCmdTrustArg ('-C') to trust the local server's self-signed certificate. Without it every connection to localhost would fail certificate validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Add .NET 10 test coverage to the CI-SqlClient pipeline Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Move CI and PR pipelines to SQL Server 2025 agent images Retire every SQL Server 2022 agent image in favour of the SQL Server 2025 equivalents: ADO-MMS22-SQL22 -> ADO-MMS25-SQL25 ADO-UB22-SQL22 -> ADO-UB24-SQL25 This covers the CI test configurations, the PR pipeline platform list, the Azure package integration test jobs (including the SQL root path, which becomes SQL25RootPath), the Managed Instance jobs, the stress test jobs, and the Linux enclave configuration. Two notes on the change: - CI test stage names derive from the image keys, so stages such as Win22_Sql22 are now named Win25_Sql25. Any branch policies or required status checks that reference the old stage names will need to be updated. - The Linux SQL configuration previously ran on both ADO-UB20-SQL22 and ADO-UB22-SQL22. We only have a single Ubuntu SQL 2025 image, so it now runs on ADO-UB24-SQL25 alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Clarify why the Linux enclave stage key is left unchanged The Linux enclave image key doubles as the generated ADO stage name and is referenced by branch policies and required status checks, so it is kept as-is. The 'Sql19' suffix also remains accurate because these tests target a remote Enclave-enabled SQL Server 2019; only the agent image moved to Ubuntu 24. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Install sqlcmd on Linux agents before configuring SQL Server The SQL Server 2025 agent images ship the engine but not the command line tools, so the Linux SQL configuration steps failed with: ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations. Add a shared 'Install sqlcmd [Linux]' step that reuses an existing sqlcmd when the image provides one and otherwise installs mssql-tools18, then publishes the resolved path via the SqlCmdBin variable. Both the PR and CI Linux configuration steps now run it, and the CI step no longer hardcodes /opt/mssql-tools/bin/sqlcmd, which does not exist on these images. sqlcmd from mssql-tools18 encrypts by default, so the step also publishes SqlCmdTrustArg ('-C') to trust the local server's self-signed certificate. Without it every connection to localhost would fail certificate validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Add .NET 10 test coverage to the CI-SqlClient pipeline Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Flow the CI pool name down from the pipeline root Review feedback: the pool name was specified in two different ways, and deep templates read it straight out of a variable group. Every CI stage and job template now takes the pool name as a required parameter, threaded down from 'defaultPoolName' in dotnet-sqlclient-ci-core.yml. That leaves exactly one reference to $(ci_var_defaultPoolName) per CI pipeline root, mirroring how the PR pipeline references $(PoolNameDefault) once at its root. Both roots now document where their variable comes from and why the two pipeline families use different variable groups. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Move CI and PR pipelines to SQL Server 2025 agent images Retire every SQL Server 2022 agent image in favour of the SQL Server 2025 equivalents: ADO-MMS22-SQL22 -> ADO-MMS25-SQL25 ADO-UB22-SQL22 -> ADO-UB24-SQL25 This covers the CI test configurations, the PR pipeline platform list, the Azure package integration test jobs (including the SQL root path, which becomes SQL25RootPath), the Managed Instance jobs, the stress test jobs, and the Linux enclave configuration. Two notes on the change: - CI test stage names derive from the image keys, so stages such as Win22_Sql22 are now named Win25_Sql25. Any branch policies or required status checks that reference the old stage names will need to be updated. - The Linux SQL configuration previously ran on both ADO-UB20-SQL22 and ADO-UB22-SQL22. We only have a single Ubuntu SQL 2025 image, so it now runs on ADO-UB24-SQL25 alone. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Clarify why the Linux enclave stage key is left unchanged The Linux enclave image key doubles as the generated ADO stage name and is referenced by branch policies and required status checks, so it is kept as-is. The 'Sql19' suffix also remains accurate because these tests target a remote Enclave-enabled SQL Server 2019; only the agent image moved to Ubuntu 24. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Install sqlcmd on Linux agents before configuring SQL Server The SQL Server 2025 agent images ship the engine but not the command line tools, so the Linux SQL configuration steps failed with: ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations. Add a shared 'Install sqlcmd [Linux]' step that reuses an existing sqlcmd when the image provides one and otherwise installs mssql-tools18, then publishes the resolved path via the SqlCmdBin variable. Both the PR and CI Linux configuration steps now run it, and the CI step no longer hardcodes /opt/mssql-tools/bin/sqlcmd, which does not exist on these images. sqlcmd from mssql-tools18 encrypts by default, so the step also publishes SqlCmdTrustArg ('-C') to trust the local server's self-signed certificate. Without it every connection to localhost would fail certificate validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Add .NET 10 test coverage to the CI-SqlClient pipeline Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Pipelines | Restore SQL Server 2022 coverage in the CI test matrix (#4587) * Restore SQL Server 2022 coverage in the CI test matrix Adds back windows_sql_22_x64 (ADO-MMS22-SQL22) and linux_ub22_sql_22 (ADO-UB22-SQL22) to the CI-SqlClient test configurations, so moving the primary configurations to SQL Server 2025 does not drop SQL Server 2022 coverage entirely. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Gate SQL Server 2022 test configurations to CI pipelines only Groups windows_sql_22_x64 and linux_ub22_sql_22 under a single runSql22Tests conditional (mirroring the existing legacy SQL block) and opts the PR pipelines out, so PR validation stays on SQL Server 2019, 2025 and Azure SQL. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Add .NET 10 test coverage to the CI-SqlClient pipeline Introduce primaryTargetFrameworks and primaryTargetFrameworksUnix, which apply only to the primary test configurations: local SQL Server 2025 and Azure SQL, on both Windows and Linux. Those configurations now run net10.0 in addition to the existing target frameworks. Restricting .NET 10 to the primary configurations keeps the added agent cost bounded rather than multiplying it across every legacy SQL Server image. The other pipelines that extend the CI core pin the new parameters to their existing target framework lists, so their behaviour is unchanged. Note that the driver itself only targets net462, net8.0, and net9.0, so the net10.0 test assemblies resolve the net9.0 driver build. These jobs therefore validate the driver on the .NET 10 runtime rather than validating a .NET 10 build of the driver. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d * Rename the Linux enclave stage key to match its Ubuntu 24 agent The key doubles as the generated stage name, and it still said Ubuntu20 after the agent moved to Ubuntu 24. A comment previously justified leaving it alone by claiming branch policies and required status checks referenced it, but that is not the case: the ADO branch policies reference pipeline definition ids, the GitHub 'Main' ruleset defines no required status check contexts, and no policy or ruleset mentions the stage name. Stage names surface only in check-run names, so the rename is safe. Rename it to Ubuntu24_Enclave_Sql19 and keep a one-line note that the 'Sql19' suffix names the remote Enclave-enabled SQL Server 2019 under test rather than the agent image. * Select agent images from the pool name in every job template Four job templates each decided differently whether to select an image with 'vmImage' (Microsoft-hosted) or an imageOverride demand (1ES): a hostedPool boolean, a poolName comparison, and a startsWith(vmImage, 'macos') check. The stress job additionally picked its own pool name from the ADO project. Unify all four on the poolName comparison already used by the package test jobs, so the invariant "the hosted 'Azure Pipelines' pool is used only for macos-latest" is visible at each call site: pool: name: ${{ parameters.poolName }} ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: vmImage: ${{ parameters.vmImage }} ${{ else }}: demands: - imageOverride -equals ${{ parameters.vmImage }} Drop the hostedPool parameter and its plumbing, give the stress job a poolName parameter flowed down from its pipeline root, and hardcode 'Azure Pipelines' at the three macOS call sites instead of routing it through an azurePoolName parameter that no caller ever set. Also rename the test configuration keys in the CI core pipeline to an _ form, and split the combined Windows Azure SQL configuration into separate Windows Server 2025 and Windows 11 configurations. * Standardize the pool parameter name and clarify pool terminology Rename the 'adoPoolName' stage parameter to 'poolName'. The name existed to distinguish it from a sibling 'azurePoolName' parameter, but that sibling is gone now that the macOS jobs name the Microsoft-hosted pool directly, and three of the five build stages that used 'adoPoolName' never had a sibling to disambiguate from. Every pool parameter in eng/pipelines is now 'poolName'. Document, on each job template that selects an image, that the pool name is compared at template-expansion time, so the Microsoft-hosted pool must be named by the literal 'Azure Pipelines' rather than a $(...) macro. Replace the ambiguous term 'hosted pool' with 'Microsoft-hosted Azure Pipelines pool', since every pool is hosted somewhere, and reword the SQL Server setup step headers from '1ES Hosted Pool' to '1ES pools'. Refresh the stale ADO-UB20-SQL22 example in the CI Linux setup step to ADO-UB24-SQL25. --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Copilot-Session: 10246002-c950-42a7-adf5-1698f9af4b3d --- .../ci/stress/sqlclient-ci-stress-job.yml | 34 ++- .../stress/sqlclient-ci-stress-pipeline.yml | 6 + .../ci/stress/sqlclient-ci-stress-stage.yml | 12 +- .../templates/jobs/ci-build-nugets-job.yml | 11 +- .../templates/jobs/ci-code-coverage-job.yml | 19 +- .../templates/jobs/ci-run-tests-job.yml | 15 +- .../templates/stages/ci-run-tests-stage.yml | 2 - .../steps/configure-sql-server-linux-step.yml | 4 +- .../steps/configure-sql-server-win-step.yml | 2 +- eng/pipelines/dotnet-sqlclient-ci-core.yml | 194 ++++++++++++++---- eng/pipelines/github-sync-pipeline.yml | 7 +- .../jobs/pack-abstractions-package-ci-job.yml | 19 +- .../jobs/pack-azure-package-ci-job.yml | 19 +- .../jobs/pack-logging-package-ci-job.yml | 19 +- .../jobs/pack-sqlserver-package-ci-job.yml | 19 +- .../jobs/test-abstractions-package-ci-job.yml | 4 + .../jobs/test-azure-package-ci-job.yml | 4 + eng/pipelines/pr/sqlclient-pr-pipeline.yml | 17 ++ .../pr/stages/collect-coverage-stage.yml | 13 +- .../pr/stages/generate-secrets-stage.yml | 16 +- eng/pipelines/pr/stages/pack-stage.yml | 13 +- .../steps/configure-sqlserver-linux-step.yml | 4 +- .../configure-sqlserver-windows-step.yml | 2 +- .../sqlclient-pr-package-ref-pipeline.yml | 3 + .../sqlclient-pr-project-ref-pipeline.yml | 3 + .../build-abstractions-package-ci-stage.yml | 24 ++- .../stages/build-azure-package-ci-stage.yml | 78 ++----- .../stages/build-logging-package-ci-stage.yml | 9 + .../build-sqlclient-package-ci-stage.yml | 9 + .../build-sqlserver-package-ci-stage.yml | 9 + .../stages/compute-versions-ci-stage.yml | 17 +- .../stages/generate-secrets-ci-stage.yml | 21 +- .../stages/verify-nuget-packages-ci-stage.yml | 19 +- 33 files changed, 476 insertions(+), 171 deletions(-) diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml index d70ce90c33..5b46f6c3fa 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml @@ -71,12 +71,17 @@ parameters: - name: sqlSetupStep type: step - # The pool VM image to use. Both the internal (ADO-1ES-Pool) and public (ADO-CI-1ES-Pool) pools - # must provide an image with this name. + # The name of the pool to run in. # - # NOTE: This value is evaluated at template-expansion (compile) time to select the pool, so it - # must be a literal and must not contain any runtime expressions (e.g. $(...) macros or - # $[...] runtime expressions). + # Supplied by the caller so that the pool name flows down from the pipeline root. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. + - name: poolName + type: string + + # The pool VM image to use, which must exist in the specified pool. - name: vmImage type: string @@ -89,11 +94,6 @@ jobs: variables: - # Whether this is an internal (ADO.Net project) or public (Public project) build. Evaluated - # at compile time so it can drive template expressions below. - - name: isInternalBuild - value: ${{ eq(variables['System.TeamProject'], 'ADO.Net') }} - # Import the variable group that provides SQL Server build properties used by the # shared configure-sql-server-*-step.yml templates (e.g. x64AliasRegistryPath, # x86AliasRegistryPath, SQLAliasName, SQLAliasPort). @@ -162,18 +162,14 @@ jobs: - name: stressTestOpts value: --assembly SqlClient.Stress.Tests --console - # Select the pool based on the requested image: - # - macOS images run on the Microsoft-hosted 'Azure Pipelines' pool (no 1ES macOS images). - # - All other images run on the 1ES pool for the current ADO project, matched via imageOverride. pool: - ${{ if startsWith(parameters.vmImage, 'macos') }}: - name: 'Azure Pipelines' + name: ${{ parameters.poolName }} + + # Images provided by Azure Pipelines must be selected using 'vmImage'. + ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: vmImage: ${{ parameters.vmImage }} + # Images provided by 1ES must be selected using a demand. ${{ else }}: - ${{ if eq(variables.isInternalBuild, true) }}: - name: ADO-1ES-Pool - ${{ else }}: - name: ADO-CI-1ES-Pool demands: - imageOverride -equals ${{ parameters.vmImage }} diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml index 74403bd743..57d52d8e4a 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml @@ -89,6 +89,9 @@ parameters: - detailed - diagnostic +variables: + - template: /eng/pipelines/libraries/ci-build-variables.yml@self + # The stages to run. stages: @@ -96,10 +99,13 @@ stages: - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self parameters: debug: ${{ parameters.debug }} + poolName: $(ci_var_defaultPoolName) + vmImage: ADO-UB24 # Run the stress tests. - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml@self parameters: + poolName: $(ci_var_defaultPoolName) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} warnOnTestFailure: ${{ parameters.warnOnTestFailure }} diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml index 3cbdb9d31e..0a1916594d 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml @@ -20,6 +20,12 @@ parameters: + # The name of the pool to use for jobs that require customized VM images. + # + # Supplied by the caller so that the pool name flows down from the pipeline root. + - name: poolName + type: string + # The type of build to produce (Debug or Release) - name: buildConfiguration type: string @@ -89,6 +95,7 @@ stages: template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self parameters: saPassword: $(saPassword) + poolName: ${{ parameters.poolName }} vmImage: ADO-UB24-SQL25 # ---------------------------------------------------------------------------------------------- @@ -112,6 +119,7 @@ stages: saPassword: $(saPassword) # The Windows images include a suitable .NET Framework runtime, so we don't have to install # one explicitly. + poolName: ${{ parameters.poolName }} vmImage: ADO-MMS25-SQL25 # ---------------------------------------------------------------------------------------------- @@ -133,5 +141,7 @@ stages: template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self parameters: saPassword: $(saPassword) - # A macOS vmImage name routes this job to the Microsoft-hosted 'Azure Pipelines' pool. + # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted + # 'Azure Pipelines' pool. + poolName: Azure Pipelines vmImage: macos-latest diff --git a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml index 95d5d0b7d0..5840663f5c 100644 --- a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml @@ -19,15 +19,18 @@ parameters: # Reference sibling packages as C# projects. - Project - # The name of Azure Pipelines pool to use. + # The name of the 1ES pool to use. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # - name: poolName type: string - default: $(ci_var_defaultPoolName) - # The name of the Azure Pipelines image to use within the pool. + # The imageOverride capability required from the 1ES pool. - name: imageOverride type: string - default: ADO-MMS22-SQL19 + default: ADO-Win25 # The name of the Abstractions pipeline artifact to download when referenceType is 'Package'. - name: abstractionsArtifactsName diff --git a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml index e4e28d4803..63c74a8013 100644 --- a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml @@ -19,13 +19,28 @@ parameters: - name: upload type: boolean + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + default: ADO-UB24 + jobs: - job: publish_code_coverage displayName: Publish Code Coverage pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} variables: # Use a temp directory that is cleaned up after each job runs. This helps diff --git a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml index 4752e09e43..ba028f1502 100644 --- a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml @@ -56,12 +56,6 @@ parameters: type: boolean default: false - # True if this job will run in a generic Azure Pipelines hosted pool; false to run in a custom 1ES - # pool. - - name: hostedPool - type: boolean - default: false - # The VM image to use, which must exist in the specified pool. - name: image type: string @@ -110,6 +104,10 @@ parameters: - Release # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string @@ -157,8 +155,11 @@ jobs: pool: name: '${{ parameters.poolName }}' - ${{ if eq(parameters.hostedPool, true) }}: + + # Images provided by Azure Pipelines must be selected using 'vmImage'. + ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: vmImage: ${{ parameters.image }} + # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - imageOverride -equals ${{ parameters.image }} diff --git a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml index 7ed67c1af9..1cda6c7580 100644 --- a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml +++ b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml @@ -106,7 +106,6 @@ stages: referenceType: ${{ parameters.referenceType }} timeout: ${{ parameters.testJobTimeout }} poolName: ${{ config.value.pool }} - hostedPool: ${{ eq(config.value.hostedPool, true) }} image: ${{ image.value }} jobDisplayName: ${{ format('{0}_{1}_{2}', replace(targetFramework, '.', '_'), platform, testSet) }} configProperties: ${{ config.value.configProperties }} @@ -137,7 +136,6 @@ stages: referenceType: ${{ parameters.referenceType }} timeout: ${{ parameters.testJobTimeout }} poolName: ${{ config.value.pool }} - hostedPool: ${{ eq(config.value.hostedPool, true) }} image: ${{ image.value }} ${{if eq(usemanagedSNI, 'true') }}: jobDisplayName: ${{ format('{0}_{1}_{2}_{3}', replace(targetFramework, '.', '_'), platform, 'ManagedSNI', testSet) }} diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml index a397108701..bafb9eda1c 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml @@ -4,8 +4,8 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This step configures an existing SQL Server running on the local Linux host. For example, our 1ES -# Hosted Pool has images like ADO-UB20-SQL22 that come with SQL Server 2022 pre-installed and +# This step configures an existing SQL Server running on the local Linux host. For example, our +# 1ES pools have images like ADO-UB24-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml index 6980c1f33c..cd691cc4a2 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml @@ -5,7 +5,7 @@ ################################################################################# # This step configures an existing SQL Server running on the local Windows host. For example, our -# 1ES Hosted Pool has images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and +# 1ES pools have images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index 335e98b575..9b53b6a85a 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -94,6 +94,18 @@ parameters: - Debug - Release + # The name of the 1ES pool that all CI jobs run in. + # + # This is the single place in the CI pipelines where the pool name is read + # from a variable group; every stage and job below receives it as a + # parameter so that the value flows down from here. + # + # 'ci_var_defaultPoolName' is defined in the 'ADO Build properties' variable + # group (see /eng/pipelines/libraries/ci-build-variables.yml), for both the + # Public and ADO.Net projects. The PR pipelines are configured from a + # different variable group and use '$(PoolNameDefault)' instead; see + # /eng/pipelines/pr/sqlclient-pr-pipeline.yml. + # - name: defaultPoolName type: string default: $(ci_var_defaultPoolName) @@ -114,6 +126,15 @@ parameters: type: boolean default: true + # If true, run manual tests against SQL Server 2022. + # + # The primary test configurations run against SQL Server 2025, so SQL Server + # 2022 is CI-only coverage. PR pipelines disable it to keep validation fast. + # + - name: runSql22Tests + type: boolean + default: true + # Build suffix appended to the prerelease tag. PR pipelines pass 'pr', # CI pipelines pass 'ci'. Official builds leave this empty. - name: buildSuffix @@ -148,16 +169,17 @@ variables: value: SqlServer.Artifacts stages: - # Compute all package versions up-front. Downstream stages consume these via - # Each build/test stage computes the versions it needs directly from the - # compute_versions_ci stage outputs, so no version parameters are passed here. + # Compute all package versions up front. Build and test stages consume the + # compute_versions_ci outputs directly, so no version parameters are passed here. - template: /eng/pipelines/stages/compute-versions-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} buildSuffix: ${{ parameters.buildSuffix }} # Generate secrets used throughout the pipeline. - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} debug: ${{ parameters.debug }} # Build the SqlServer package, and publish it to the pipeline artifacts @@ -165,6 +187,7 @@ stages: # generation stage since it has no package dependencies. - template: /eng/pipelines/stages/build-sqlserver-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} sqlServerArtifactsName: $(sqlServerArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} @@ -175,6 +198,7 @@ stages: # generation stage since it has no package dependencies. - template: /eng/pipelines/stages/build-logging-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} loggingArtifactsName: $(loggingArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} @@ -187,6 +211,7 @@ stages: # Abstractions has a package dependency on Logging. - template: /eng/pipelines/stages/build-abstractions-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} @@ -205,6 +230,7 @@ stages: # - template: /eng/pipelines/stages/build-sqlclient-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} loggingArtifactsName: $(loggingArtifactsName) @@ -225,6 +251,7 @@ stages: # given artifact name. - template: /eng/pipelines/stages/build-azure-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) azureArtifactsName: $(azureArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} @@ -248,6 +275,7 @@ stages: # published as pipeline artifacts. - template: /eng/pipelines/stages/verify-nuget-packages-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) azureArtifactsName: $(azureArtifactsName) loggingArtifactsName: $(loggingArtifactsName) @@ -295,6 +323,7 @@ stages: - template: /eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml@self parameters: debug: ${{ parameters.debug }} + poolName: ${{ parameters.defaultPoolName }} # We only want to upload coverage results to CodeCov from certain # pipelines. We use the pipeline name (Build.DefinitionName) to # choose. This is a predefined variable that is available at @@ -316,13 +345,14 @@ stages: # Configuration of test jobs. Each entry in this object will become a test job, and the # properties of each entry will be supplied as parameters to the test job template. + # + # The OS architecture is assumed to be x64 unless otherwise noted. + # testConfigurations: - # SQL Server 2016 and 2017 on Windows Server 2022 (x64 only). - # x86 testing is intentionally skipped for these legacy SQL versions - # because x86 support is already validated via SQL 2019 and 2025 images. + # SQL Server 2016 and 2017 on Windows Server 2022. ${{ if eq(parameters.runLegacySqlTests, true) }}: - # Windows Server 22 with local SQL Server 2016, x64 build platform. - windows_sql_16_x64: + # Windows Server 22 with local SQL Server 2016. + win22_sql16: pool: ${{parameters.defaultPoolName }} images: Win22_Sql16: ADO-MMS22-SQL16 @@ -347,8 +377,8 @@ stages: SQLRootPath: $(SQL16RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2017, x64 build platform. - windows_sql_17_x64: + # Windows Server 22 with local SQL Server 2017. + win22_sql17: pool: ${{parameters.defaultPoolName }} images: Win22_Sql17: ADO-MMS22-SQL17 @@ -373,8 +403,8 @@ stages: SQLRootPath: $(SQL17RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2019, x64 build platform. - windows_sql_19_x64: + # Windows Server 22 with local SQL Server 2019. + win22_sql19: pool: ${{parameters.defaultPoolName }} images: Win22_Sql19: ADO-MMS22-SQL19 @@ -400,8 +430,8 @@ stages: SQLRootPath: $(SQL19RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2019, x86 build platform. - windows_sql_19_x86: + # Windows Server 22 with local SQL Server 2019, x86. + win22_sql19_x86: pool: ${{parameters.defaultPoolName }} images: Win22_Sql19_x86: ADO-MMS22-SQL19 @@ -427,8 +457,8 @@ stages: SQLRootPath: $(SQL19RootPath) enableLocalDB: true - # Windows Server 25 with local SQL Server 2025, x64 build platform. - windows_sql_25_x64: + # Windows Server 25 with local SQL Server 2025. + win25_sql25: pool: ${{parameters.defaultPoolName }} images: Win25_Sql25: ADO-MMS25-SQL25 @@ -453,8 +483,8 @@ stages: SQLRootPath: $(SQL25RootPath) enableLocalDB: true - # Windows Server 25 with local SQL Server 2025, x86 build platform. - windows_sql_25_x86: + # Windows Server 25 with local SQL Server 2025, x86. + win25_sql25_x86: pool: ${{parameters.defaultPoolName }} images: Win25_Sql25_x86: ADO-MMS25-SQL25 @@ -480,8 +510,8 @@ stages: SQLRootPath: $(SQL25RootPath) enableLocalDB: true - # Windows Server 25 with local SQL Server 2025 Named Instance, x64 build platform. - windows_sql_25_named_instance: + # Windows Server 25 with local SQL Server 2025 Named Instance. + win25_sql25_named_instance: pool: ${{parameters.defaultPoolName }} images: Win25_Sql25_Named_Instance: ADO-MMS25-SQL25-WITH-NAMED-INSTANCE @@ -499,12 +529,11 @@ stages: SQLRootPath: $(SQL25RootPath) instanceName: $(NamedInstance) - # Windows Server 2022 and Windows 11, x64 build platform, with Azure SQL Server. - windows_azure_sql: + # Windows Server 2025, x64 build platform, with Azure SQL Server. + win25_azure_sql: pool: ${{parameters.defaultPoolName }} images: - Win22_Azure_Sql: ADO-MMS22-SQL19 - Win11_Azure_Sql: ADO-CI-Win11 + Win25_Azure_Sql: ADO-Win25 TargetFrameworks: ${{parameters.primaryTargetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -530,11 +559,38 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + # Windows 11, x64 build platform, with Azure SQL Server. + win11_azure_sql: + pool: ${{parameters.defaultPoolName }} + images: + Win11_Azure_Sql: ADO-CI-Win11 + TargetFrameworks: ${{parameters.targetFrameworks }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: ${{parameters.buildPlatforms }} + testSets: ${{parameters.testSets }} + useManagedSNI: ${{parameters.useManagedSNI }} + configSqlFor: azure + operatingSystem: Windows + configProperties: + TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING) + NPConnectionString: $(AZURE_DB_NP_CONN_STRING) + AADAuthorityURL: $(AADAuthorityURL) + ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: + AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR) + AADServicePrincipalSecret: $(AADServicePrincipalSecret) + AADServicePrincipalId: $(AADServicePrincipalId) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: false + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + # Windows 11 on ARM64 with Azure SQL Server. - windows_azure_arm64_sql: + win11_azure_sql_arm64: pool: ADO-CI-PUBLIC-ARM64-1ES-EUS-POOL images: - Win11_ARM64_Azure_Sql: ADO-WIN11-ARM64 + Win11_Azure_Sql_ARM64: ADO-WIN11-ARM64 TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -557,8 +613,8 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # Linux Ubuntu 24 with local SQL Server 2025, x64 build platform. - linux_ub24_sql_25: + # Linux Ubuntu 24 with local SQL Server 2025. + linux_ub24_sql25: pool: ${{parameters.defaultPoolName }} images: Ubuntu24_Sql25: ADO-UB24-SQL25 @@ -580,11 +636,67 @@ stages: LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) - # Linux Ubuntu 24 with Azure SQL Server, x64 build platform. - linux_azure_sql: + # SQL Server 2022 coverage, on Windows and Linux. + # + # The primary configurations run against SQL Server 2025, so these keep + # SQL Server 2022 in the matrix. They are grouped under a single + # conditional so that PR pipelines can opt out of them as a unit. + ${{ if eq(parameters.runSql22Tests, true) }}: + # Windows Server 22 with local SQL Server 2022. + win22_sql22: + pool: ${{parameters.defaultPoolName }} + images: + Win22_Sql22: ADO-MMS22-SQL22 + TargetFrameworks: ${{parameters.targetFrameworks }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: ${{parameters.buildPlatforms }} + testSets: ${{parameters.testSets }} + useManagedSNI: ${{parameters.useManagedSNI }} + configSqlFor: local + operatingSystem: Windows + # config.jsonc properties + configProperties: + TCPConnectionString: $(SQL_TCP_CONN_STRING) + NPConnectionString: $(SQL_NP_CONN_STRING) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: true + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + FileStreamDirectory: $(FileStreamDirectory) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + AliasName: $(SQLAliasName) + SQLRootPath: $(SQL22RootPath) + enableLocalDB: true + + # Linux Ubuntu 22 with local SQL Server 2022. + linux_ub22_sql22: + pool: ${{parameters.defaultPoolName }} + images: + Ubuntu22_Sql22: ADO-UB22-SQL22 + TargetFrameworks: ${{parameters.targetFrameworksUnix }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: [AnyCPU] + testSets: ${{parameters.testSets }} + useManagedSNI: [true] + configSqlFor: local + operatingSystem: Linux + configProperties: + TCPConnectionString: $(SQL_TCP_CONN_STRING) + NPConnectionString: $(SQL_NP_CONN_STRING) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: false + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + AliasName: $(SQLAliasName) + + # Linux Ubuntu 24 with Azure SQL Server. + linux_ub24_azure_sql: pool: ${{parameters.defaultPoolName }} images: - Ubuntu24_Azure_Sql: ADO-UB24-SQL25 + Ubuntu24_Azure_Sql: ADO-UB24 TargetFrameworks: ${{parameters.primaryTargetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] @@ -608,9 +720,8 @@ stages: LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) # macOS with local SQL Server 2025 (docker), x64 build platform. - mac_sql_25: + mac_sql25: pool: Azure Pipelines - hostedPool: true images: MacOSLatest_Sql25: macos-latest TargetFrameworks: ${{parameters.targetFrameworksUnix }} @@ -633,11 +744,11 @@ stages: # Only run these tests if explicitly enabled, and if we're not a forked repo (which won't # have access to the necessary Library secrets). ${{ if and(eq(parameters.runAlwaysEncryptedTests, true), eq(variables['System.PullRequest.IsFork'], 'False')) }}: - # Windows Server 22 with remote Enclave-enabled SQL Server 2019, x64 build platform. - windows_enclave_sql: + # Windows Server 22 with remote Enclave-enabled SQL Server 2019. + win22_enclave_sql19: pool: ADO-CI-AE-1ES-Pool images: - Win22_Enclave_Sql19: ADO-MMS22-SQL19 + Win22_Enclave_Sql19: ADO-Win25 TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -661,16 +772,11 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # Linux Ubuntu 24 with remote Enclave-enabled SQL Server 2019, x64 build platform. - linux_enclave_sql: + # Linux Ubuntu 24 with remote Enclave-enabled SQL Server 2019. + linux_ub24_enclave_sql19: pool: ADO-CI-AE-1ES-Pool images: - # NOTE: This key is also the generated stage name, and is - # referenced by branch policies / required status checks, so it is - # deliberately left unchanged. The 'Sql19' suffix remains - # accurate: these tests target a remote Enclave-enabled SQL Server - # 2019. Only the agent image has moved to Ubuntu 24. - Ubuntu20_Enclave_Sql19: ADO-UB24-SQL25 + Ubuntu24_Enclave_Sql19: ADO-UB24 TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] diff --git a/eng/pipelines/github-sync-pipeline.yml b/eng/pipelines/github-sync-pipeline.yml index 699270f4de..6d186a10b1 100644 --- a/eng/pipelines/github-sync-pipeline.yml +++ b/eng/pipelines/github-sync-pipeline.yml @@ -57,11 +57,16 @@ parameters: type: string default: internal/main +variables: + - template: /eng/pipelines/libraries/ci-build-variables.yml@self + jobs: - job: SyncGitHub displayName: Sync GitHub to ADO pool: - vmImage: 'ubuntu-latest' + name: $(ci_var_defaultPoolName) + demands: + - imageOverride -equals ADO-UB24 steps: # Check out the ADO repo with full history so we can compare branches. diff --git a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml index 8bf43bd997..bbd80cc3f9 100644 --- a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml @@ -67,6 +67,19 @@ parameters: # Reference sibling packages as C# projects. - Project + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + default: ADO-UB24 + jobs: - job: pack_abstractions_package_job @@ -75,8 +88,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} variables: diff --git a/eng/pipelines/jobs/pack-azure-package-ci-job.yml b/eng/pipelines/jobs/pack-azure-package-ci-job.yml index 38eaf58fe9..b626dd14a5 100644 --- a/eng/pipelines/jobs/pack-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-azure-package-ci-job.yml @@ -73,6 +73,19 @@ parameters: # Reference sibling packages as C# projects. - Project + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + default: ADO-UB24 + jobs: - job: pack_azure_package_job @@ -81,8 +94,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} variables: diff --git a/eng/pipelines/jobs/pack-logging-package-ci-job.yml b/eng/pipelines/jobs/pack-logging-package-ci-job.yml index 0f0faa1859..03fde36308 100644 --- a/eng/pipelines/jobs/pack-logging-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-logging-package-ci-job.yml @@ -50,6 +50,19 @@ parameters: - detailed - diagnostic + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + default: ADO-UB24 + jobs: - job: pack_logging_package_job @@ -58,8 +71,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} variables: diff --git a/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml b/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml index 4f619b3f1b..c869026812 100644 --- a/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml @@ -49,6 +49,19 @@ parameters: - detailed - diagnostic + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + default: ADO-UB24 + jobs: - job: pack_sqlserver_package_job @@ -57,8 +70,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} variables: diff --git a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml index 83366b52a9..1cdf04c0be 100644 --- a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml @@ -58,6 +58,10 @@ parameters: default: [] # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string diff --git a/eng/pipelines/jobs/test-azure-package-ci-job.yml b/eng/pipelines/jobs/test-azure-package-ci-job.yml index 104401b585..cedd26e056 100644 --- a/eng/pipelines/jobs/test-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/test-azure-package-ci-job.yml @@ -98,6 +98,10 @@ parameters: default: [] # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string diff --git a/eng/pipelines/pr/sqlclient-pr-pipeline.yml b/eng/pipelines/pr/sqlclient-pr-pipeline.yml index 4ab1a31f60..5ad15f5474 100644 --- a/eng/pipelines/pr/sqlclient-pr-pipeline.yml +++ b/eng/pipelines/pr/sqlclient-pr-pipeline.yml @@ -103,6 +103,17 @@ variables: - template: /eng/pipelines/pr/variables/pr-variables.yml@self stages: + # NOTE: '$(PoolNameDefault)' comes from the 'sqlclient-testconfig-v1' + # variable group, which is imported by + # /eng/pipelines/pr/variables/pr-variables.yml (included above). It is + # referenced here, at the pipeline root, and passed down to every stage as a + # parameter, so that no template reads the pool name from a variable group + # directly. + # + # The CI pipelines are configured from a different variable group and use + # '$(ci_var_defaultPoolName)' instead; see + # /eng/pipelines/dotnet-sqlclient-ci-core.yml. + # Stage 1a: Build and pack all projects in the repository - template: /eng/pipelines/pr/stages/pack-stage.yml@self parameters: @@ -110,11 +121,15 @@ stages: buildSuffix: pr stageName: ${{ variables.stageNamePack }} packArtifactBaseName: ${{ variables.packArtifactBaseName }} + poolName: $(PoolNameDefault) + vmImage: ADO-UB24 # Stage 1b: Generate secrets - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml@self parameters: stageName: ${{ variables.stageNameSecrets }} + poolName: $(PoolNameDefault) + vmImage: ADO-UB24 # Stage 2: Execute tests and collect code coverage - template: /eng/pipelines/pr/stages/test-stages.yml@self @@ -144,6 +159,8 @@ stages: - template: /eng/pipelines/pr/stages/collect-coverage-stage.yml@self parameters: coverageArtifactBaseName: ${{ variables.coverageArtifactBaseName }} + poolName: $(PoolNameDefault) + vmImage: ADO-UB24 dependsOn: - ${{ each platform in parameters.platforms }}: - "test_${{ platform.displayName }}" diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml index 9a0328ec50..a81b4bb1be 100644 --- a/eng/pipelines/pr/stages/collect-coverage-stage.yml +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -16,6 +16,14 @@ parameters: - name: coverageArtifactBaseName type: string + # The name of the 1ES pool to run in. + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + stages: - stage: collect_code_coverage displayName: "Collect code coverage" @@ -26,7 +34,10 @@ stages: displayName: "Collect Code Coverage" pool: - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} variables: diff --git a/eng/pipelines/pr/stages/generate-secrets-stage.yml b/eng/pipelines/pr/stages/generate-secrets-stage.yml index 20a81fe626..bd7d2d6a5c 100644 --- a/eng/pipelines/pr/stages/generate-secrets-stage.yml +++ b/eng/pipelines/pr/stages/generate-secrets-stage.yml @@ -30,6 +30,14 @@ parameters: - name: stageName type: string + # The name of the 1ES pool to run in. + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + stages: # The stage downstream stages must depend on to ensure the secrets are generated before they are @@ -45,9 +53,11 @@ stages: - job: secrets_job displayName: Generate Secrets pool: - # We don't need anything special, so use the standard Microsoft-hosted Ubuntu image, which - # is typically very fast to spin up. - vmImage: ubuntu-latest + # This job has no special image requirements, so use a minimal Linux image. + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} steps: diff --git a/eng/pipelines/pr/stages/pack-stage.yml b/eng/pipelines/pr/stages/pack-stage.yml index 2b43c8632a..8da436f445 100644 --- a/eng/pipelines/pr/stages/pack-stage.yml +++ b/eng/pipelines/pr/stages/pack-stage.yml @@ -42,6 +42,14 @@ parameters: - name: packArtifactBaseName type: string + # The name of the 1ES pool to run in. + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + stages: - stage: ${{ parameters.stageName }} displayName: Build and Pack Projects @@ -52,7 +60,10 @@ stages: displayName: Build and Pack Projects pool: - vmImage: 'ubuntu-latest' + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} steps: # Install dotnet diff --git a/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml b/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml index df625dc35e..c240b8525f 100644 --- a/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml +++ b/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml @@ -4,8 +4,8 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This step configures an existing SQL Server running on the local Linux host. For example, our 1ES -# Hosted Pool has images like ADO-UB24-SQL25 that come with SQL Server 2025 pre-installed and +# This step configures an existing SQL Server running on the local Linux host. For example, our +# 1ES pools have images like ADO-UB24-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: diff --git a/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml b/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml index 405582d655..cf66c1e9c3 100644 --- a/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml +++ b/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml @@ -5,7 +5,7 @@ ################################################################################# # This step configures an existing SQL Server running on the local Windows host. For example, our -# 1ES Hosted Pool has images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and +# 1ES pools have images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: diff --git a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml index abb0ef5f39..7623f83ce4 100644 --- a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml @@ -145,6 +145,9 @@ extends: useManagedSNI: ${{ parameters.useManagedSNI }} # Legacy SQL Server tests (2016/2017) run in CI only, not on PRs. runLegacySqlTests: false + # SQL Server 2022 coverage runs in CI only. PR validation covers SQL + # Server 2019, 2025 and Azure SQL. + runSql22Tests: false # Don't run the AE tests in Debug mode; they rarely succeed. ${{ if eq(parameters.buildConfiguration, 'Debug') }}: runAlwaysEncryptedTests: false diff --git a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml index f9578e6548..74309ae643 100644 --- a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml @@ -145,6 +145,9 @@ extends: useManagedSNI: ${{ parameters.useManagedSNI }} # Legacy SQL Server tests (2016/2017) run in CI only, not on PRs. runLegacySqlTests: false + # SQL Server 2022 coverage runs in CI only. PR validation covers SQL + # Server 2019, 2025 and Azure SQL. + runSql22Tests: false # Don't run the AE tests in Debug mode; they rarely succeed. ${{ if eq(parameters.buildConfiguration, 'Debug') }}: runAlwaysEncryptedTests: false diff --git a/eng/pipelines/stages/build-abstractions-package-ci-stage.yml b/eng/pipelines/stages/build-abstractions-package-ci-stage.yml index 3df0ef9843..3342bfca01 100644 --- a/eng/pipelines/stages/build-abstractions-package-ci-stage.yml +++ b/eng/pipelines/stages/build-abstractions-package-ci-stage.yml @@ -35,6 +35,19 @@ parameters: type: object default: [] + # The name of the pool to use for jobs that require customized VM images. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + # Any pool specified here must contain images with the following names: + # + # - ADO-Win25 + # - ADO-UB24 + # + - name: poolName + type: string + # The type of build to produce (Release or Debug) - name: buildConfiguration type: string @@ -105,8 +118,8 @@ stages: jobNameSuffix: linux netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] - poolName: Azure Pipelines - vmImage: ubuntu-latest + poolName: ${{ parameters.poolName }} + vmImage: ADO-UB24 # ------------------------------------------------------------------------ # Build and test on Windows @@ -120,8 +133,8 @@ stages: jobNameSuffix: windows netFrameworkRuntimes: [net462] netRuntimes: [net8.0, net9.0, net10.0] - poolName: Azure Pipelines - vmImage: windows-latest + poolName: ${{ parameters.poolName }} + vmImage: ADO-Win25 # ------------------------------------------------------------------------ # Build and test on macOS. @@ -135,6 +148,8 @@ stages: jobNameSuffix: macos netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] + # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted + # 'Azure Pipelines' pool. poolName: Azure Pipelines vmImage: macos-latest @@ -143,6 +158,7 @@ stages: - template: /eng/pipelines/jobs/pack-abstractions-package-ci-job.yml@self parameters: + poolName: ${{ parameters.poolName }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} # The version is computed by this stage (see the packageVersion variable above). packageVersion: $(packageVersion) diff --git a/eng/pipelines/stages/build-azure-package-ci-stage.yml b/eng/pipelines/stages/build-azure-package-ci-stage.yml index b907d2fcda..79930f665e 100644 --- a/eng/pipelines/stages/build-azure-package-ci-stage.yml +++ b/eng/pipelines/stages/build-azure-package-ci-stage.yml @@ -40,29 +40,23 @@ parameters: default: [] # The name of the pool to use for jobs that require customized VM images. - - name: adoPoolName + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + # Any pool specified here must contain images with the following names: + # + # - ADO-MMS25-SQL25 + # - ADO-UB24-SQL25 + # + - name: poolName type: string - # This variable should be defined in AzureDevOps Library variable groups, - # for both the Public and ADO.Net projects. - # - # Any pool specified here must contain images with the following names: - # - # - ADO-MMS25-SQL25 - # - ADO-UB24-SQL25 - # - default: $(ci_var_defaultPoolName) # The name of the pipeline artifacts to publish. - name: azureArtifactsName type: string default: Azure.Artifacts - # The name of the general Azure pool to use for jobs that don't require - # customized VM images. - - name: azurePoolName - type: string - default: Azure Pipelines - # The type of build to produce (Release or Debug) - name: buildConfiguration type: string @@ -144,25 +138,6 @@ stages: # ------------------------------------------------------------------------ # Build and test on Linux. - - template: /eng/pipelines/jobs/test-azure-package-ci-job.yml@self - parameters: - abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} - packageVersion: $(packageVersion) - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayNamePrefix: Linux - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - jobNameSuffix: linux - loggingArtifactsName: ${{ parameters.loggingArtifactsName }} - mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} - sqlServerPackageVersion: $(sqlServerPackageVersion) - netFrameworkRuntimes: [] - netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.azurePoolName }} - referenceType: ${{ parameters.referenceType }} - vmImage: ubuntu-latest - # Use our 1ES ADO pool for comprehensive testing. - template: /eng/pipelines/jobs/test-azure-package-ci-job.yml@self parameters: @@ -179,7 +154,7 @@ stages: sqlServerPackageVersion: $(sqlServerPackageVersion) netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.adoPoolName }} + poolName: ${{ parameters.poolName }} referenceType: ${{ parameters.referenceType }} saPassword: $(saPassword) # The image includes a SQL Server instance that we must configure. @@ -192,26 +167,6 @@ stages: # ------------------------------------------------------------------------ # Build and test on Windows - # Use the Azure Pipelines pool for basic testing. - - template: /eng/pipelines/jobs/test-azure-package-ci-job.yml@self - parameters: - abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} - packageVersion: $(packageVersion) - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayNamePrefix: Win - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - jobNameSuffix: windows - loggingArtifactsName: ${{ parameters.loggingArtifactsName }} - mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} - sqlServerPackageVersion: $(sqlServerPackageVersion) - netFrameworkRuntimes: [net462] - netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.azurePoolName }} - referenceType: ${{ parameters.referenceType }} - vmImage: windows-latest - # Use our 1ES ADO pool for comprehensive testing. - template: /eng/pipelines/jobs/test-azure-package-ci-job.yml@self parameters: @@ -228,7 +183,7 @@ stages: sqlServerPackageVersion: $(sqlServerPackageVersion) netFrameworkRuntimes: [net462] netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.adoPoolName }} + poolName: ${{ parameters.poolName }} referenceType: ${{ parameters.referenceType }} saPassword: $(saPassword) # The image includes a SQL Server instance that we must configure. @@ -266,17 +221,18 @@ stages: sqlServerPackageVersion: $(sqlServerPackageVersion) netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.azurePoolName }} + # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted + # 'Azure Pipelines' pool. + poolName: Azure Pipelines referenceType: ${{ parameters.referenceType }} vmImage: macos-latest - # We do not currently have any images in our 1ES ADO pools for macOS. - # ------------------------------------------------------------------------ # Create and publish the NuGet package. - template: /eng/pipelines/jobs/pack-azure-package-ci-job.yml@self parameters: + poolName: ${{ parameters.poolName }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} packageVersion: $(packageVersion) azureArtifactsName: ${{ parameters.azureArtifactsName }} @@ -285,9 +241,7 @@ stages: dependsOn: # We depend on all of the test jobs to ensure the tests pass before # producing the NuGet package. - - test_azure_package_job_linux - test_azure_package_job_linux_integration - - test_azure_package_job_windows - test_azure_package_job_windows_integration - test_azure_package_job_macos dotnetVerbosity: ${{ parameters.dotnetVerbosity }} diff --git a/eng/pipelines/stages/build-logging-package-ci-stage.yml b/eng/pipelines/stages/build-logging-package-ci-stage.yml index b9d5feb082..9b421b821a 100644 --- a/eng/pipelines/stages/build-logging-package-ci-stage.yml +++ b/eng/pipelines/stages/build-logging-package-ci-stage.yml @@ -25,6 +25,14 @@ parameters: + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + # Additional stages we depend on, if any. - name: additionalDependsOn type: object @@ -84,6 +92,7 @@ stages: - template: /eng/pipelines/jobs/pack-logging-package-ci-job.yml@self parameters: + poolName: ${{ parameters.poolName }} loggingArtifactsName: ${{ parameters.loggingArtifactsName }} # The version is computed by this stage (see the packageVersion variable above). packageVersion: $(packageVersion) diff --git a/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml b/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml index 70fa122d5f..1895f4e593 100644 --- a/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml +++ b/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml @@ -18,6 +18,14 @@ parameters: type: string default: Abstractions.Artifacts + # The name of the 1ES pool to run in. Hosted Azure Pipelines pools are not supported. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + # Additional stages we depend on, if any. - name: additionalDependsOn type: object @@ -84,6 +92,7 @@ stages: jobs: - template: /eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml@self parameters: + poolName: ${{ parameters.poolName }} buildConfiguration: ${{ parameters.buildConfiguration }} referenceType: ${{ parameters.referenceType }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} diff --git a/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml b/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml index 3a1417185d..633e3bc3f1 100644 --- a/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml +++ b/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml @@ -25,6 +25,14 @@ parameters: + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + # Additional stages we depend on, if any. - name: additionalDependsOn type: object @@ -77,6 +85,7 @@ stages: jobs: - template: /eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml@self parameters: + poolName: ${{ parameters.poolName }} buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} diff --git a/eng/pipelines/stages/compute-versions-ci-stage.yml b/eng/pipelines/stages/compute-versions-ci-stage.yml index 9171582cd2..d508ba51ed 100644 --- a/eng/pipelines/stages/compute-versions-ci-stage.yml +++ b/eng/pipelines/stages/compute-versions-ci-stage.yml @@ -19,6 +19,19 @@ # This stage MUST run before all build stages so they can consume computed versions. parameters: + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + default: ADO-UB24 + # Build suffix appended to prerelease tag (e.g. 'ci' or 'pr'). - name: buildSuffix type: string @@ -32,7 +45,9 @@ stages: - job: compute_versions_job displayName: "Extract versions from Versions.props" pool: - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + demands: + - imageOverride -equals ${{ parameters.vmImage }} steps: # Install the global.json-pinned .NET SDK before running any dotnet build, so this stage diff --git a/eng/pipelines/stages/generate-secrets-ci-stage.yml b/eng/pipelines/stages/generate-secrets-ci-stage.yml index eaf61e1f60..d6f38f8f85 100644 --- a/eng/pipelines/stages/generate-secrets-ci-stage.yml +++ b/eng/pipelines/stages/generate-secrets-ci-stage.yml @@ -31,6 +31,19 @@ parameters: type: boolean default: false + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + default: ADO-UB24 + stages: # The stage downstream stages must depend on to ensure the secrets are generated before they are @@ -48,9 +61,11 @@ stages: - job: secrets_job displayName: Generate Secrets pool: - # We don't need anything special, so use the standard Microsoft-hosted Ubuntu image, which - # is typically very fast to spin up. - vmImage: ubuntu-latest + # This job has no special image requirements, so use a minimal Linux image. + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} steps: diff --git a/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml b/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml index 3dccd61989..06d43769fd 100644 --- a/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml +++ b/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml @@ -38,6 +38,19 @@ parameters: type: string default: SqlServer.Artifacts + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: vmImage + type: string + default: ADO-Win25 + stages: - stage: verify_nuget_packages_stage @@ -55,8 +68,10 @@ stages: displayName: Verify NuGet Package Metadata pool: - name: Azure Pipelines - vmImage: windows-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.vmImage }} variables: # The directory where all package artifacts will be downloaded. From ce9f01b8b1583b2ab80b51de7021b3c01e1298bc Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:36:10 -0300 Subject: [PATCH 23/51] Install the macOS docker CLI from a Homebrew bottle instead of source-building it (#4661) --- .../steps/configure-sql-server-macos-step.yml | 48 ++-- .../scripts/Install-DockerCli.macos.ps1 | 225 ++++++++++++++++++ .../tests/Install-DockerCli.macos.Tests.ps1 | 194 +++++++++++++++ eng/pipelines/scripts/tests/README.md | 5 +- 4 files changed, 455 insertions(+), 17 deletions(-) create mode 100644 eng/pipelines/scripts/Install-DockerCli.macos.ps1 create mode 100644 eng/pipelines/scripts/tests/Install-DockerCli.macos.Tests.ps1 diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml index 89b5e9f738..772433ba3a 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml @@ -29,21 +29,33 @@ steps: export PS4='+ [$(date "+%Y-%m-%d %H:%M:%S")] ' set -x - # Install Docker CLI (not Desktop — Colima provides the daemon) and SQLCMD tools. + # Install Colima (which provides the Docker daemon, since Docker Desktop is + # not available here), the docker CLI, and sqlcmd. brew install colima - brew install docker - brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release - brew update - # Homebrew 5.2+ requires explicit trust for third-party taps. Run this - # after 'brew update' so the trust command is available even if the runner - # image shipped an older Homebrew version. - brew trust microsoft/mssql-release - HOMEBREW_ACCEPT_EULA=Y brew install mssql-tools18 - - # Fail fast if sqlcmd was not installed (e.g. tap-trust or formula error). - # Without this check the script would loop for ~6 minutes trying to connect. + + # Homebrew ships no Intel macOS bottle for the current docker formula, so + # 'brew install docker' compiles the CLI (and builds Go to do it), which + # takes minutes and often exhausts the step timeout. Install the newest + # bottled version directly instead. + DOCKER_CLI_DIR="$HOME/.docker-cli/bin" + if ! pwsh -NoProfile -File "$(Build.SourcesDirectory)/eng/pipelines/scripts/Install-DockerCli.macos.ps1" -DestinationPath "$DOCKER_CLI_DIR"; then + echo "ERROR: Failed to install the docker CLI." + exit 1 + fi + # prependpath only affects later steps, so fix PATH for this one as well. + export PATH="$DOCKER_CLI_DIR:$PATH" + + # go-sqlcmd, rather than mssql-tools18 from the microsoft/mssql-release + # tap. That tap pins an openssl@3 formula that has no Intel bottle, so + # installing it compiled openssl from source and cost 12.5 minutes of the + # step budget. go-sqlcmd is a single bottled Go binary with no openssl + # dependency, and accepts the same flags used below. + brew install sqlcmd + + # Fail fast if sqlcmd was not installed. Without this check the script + # would loop for ~6 minutes trying to connect. if ! command -v sqlcmd &>/dev/null; then - echo "ERROR: sqlcmd is not on PATH after brew install. Check the mssql-tools18 installation above." + echo "ERROR: sqlcmd is not on PATH after 'brew install sqlcmd'." exit 1 fi @@ -82,8 +94,8 @@ steps: fi # Point the docker CLI at Colima's daemon socket. Colima normally sets an - # active docker context, but the standalone docker CLI (installed above via - # 'brew install docker') can default to unix:///var/run/docker.sock, which + # active docker context, but the standalone docker CLI installed above can + # default to unix:///var/run/docker.sock, which # does not exist on macOS without Docker Desktop. This caused every # 'docker pull' to fail instantly with: # failed to connect to the docker API at unix:///var/run/docker.sock @@ -231,3 +243,9 @@ steps: fi displayName: 'Configure SQL Server [macOS]' + # Well above a healthy run, but low enough that a wedged install or Colima + # boot fails here instead of consuming the whole test job. Measured worst + # case is ~24 minutes: Colima boot ~6, the SQL image pull ~10 (7 of which is + # extraction inside the VM), and up to 6 more waiting for SQL to accept + # connections. + timeoutInMinutes: 40 diff --git a/eng/pipelines/scripts/Install-DockerCli.macos.ps1 b/eng/pipelines/scripts/Install-DockerCli.macos.ps1 new file mode 100644 index 0000000000..bdaf588e5c --- /dev/null +++ b/eng/pipelines/scripts/Install-DockerCli.macos.ps1 @@ -0,0 +1,225 @@ +<# +.SYNOPSIS + Installs the docker CLI on an Intel macOS agent from a Homebrew bottle. + +.DESCRIPTION + Homebrew ships no Intel macOS bottle for the current docker formula, so + 'brew install docker' compiles the CLI - and builds Go in order to do it - + which takes minutes and routinely exhausts the pipeline step timeout. + + This installs the newest docker version that *is* bottled for Intel macOS, + read from Homebrew's own OCI registry on ghcr.io. Bottles are + content-addressed, so the download is verified against the digest the + registry advertises rather than a checksum pinned here that someone has to + remember to bump. Nothing is downloaded from outside Homebrew, and the + result is exactly what 'brew install' would have produced. + + Only the docker CLI is handled this way. colima and lima still install + through brew: their bottles carry payloads outside bin/ and colima depends + on lima at runtime, so neither can be installed by lifting a single binary. + +.PARAMETER DestinationPath + Directory the docker binary is written to. Prepended to PATH for subsequent + pipeline steps. + +.NOTES + Intel macOS only. On any other architecture this fails rather than + installing a binary the agent cannot run. +#> +param( + [string]$DestinationPath = (Join-Path -Path $HOME -ChildPath '.docker-cli/bin') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# tar's exit code is checked explicitly below so the failure carries the member +# path rather than PowerShell's generic native-command error. +$PSNativeCommandUseErrorActionPreference = $false + +$Registry = 'https://ghcr.io' +$RegistryRepository = 'homebrew/core/docker' + +# Newest first. A bottle built for an older macOS still runs on a newer one, so +# any of these work on the current agents. +$PreferredCodenames = @('sequoia', 'sonoma', 'ventura') + +#region Helper Functions + +function Get-RegistryToken { + <# + .SYNOPSIS + An anonymous pull token for the formula's registry repository. + #> + + $uri = "$Registry/token?service=ghcr.io&scope=repository:${RegistryRepository}:pull" + return (Invoke-RestMethod -Uri $uri -MaximumRetryCount 3 -RetryIntervalSec 5).token +} + +function Get-BottleVersion { + <# + .SYNOPSIS + Every docker version tag in the registry, newest first. + #> + param( + [Parameter(Mandatory)][string]$Token + ) + + $uri = "$Registry/v2/$RegistryRepository/tags/list?n=1000" + $tags = @() + + while ($uri) { + $response = Invoke-RestMethod ` + -Uri $uri ` + -Headers @{ Authorization = "Bearer $Token" } ` + -MaximumRetryCount 3 -RetryIntervalSec 5 ` + -ResponseHeadersVariable 'responseHeaders' + + $tags += $response.tags + + # ghcr.io orders tags lexically, which puts the newest on the last page. + $link = if ($responseHeaders.ContainsKey('Link')) { @($responseHeaders['Link'])[0] } else { $null } + $uri = if ($link -match '<([^>]+)>') { $Registry + $Matches[1] } else { $null } + } + + return $tags | + Where-Object { $_ -match '^\d+\.\d+\.\d+(-\d+)?$' } | + Sort-Object -Property { [version]($_ -replace '-', '.') } -Descending +} + +function Get-BottleRefName { + <# + .SYNOPSIS + The bottle ref name an OCI index entry advertises, or $null. + #> + param( + [Parameter(Mandatory)]$IndexEntry + ) + + $annotations = $IndexEntry.PSObject.Properties['annotations'] + if (-not $annotations) { return $null } + + $refName = $annotations.Value.PSObject.Properties['org.opencontainers.image.ref.name'] + if (-not $refName) { return $null } + + return $refName.Value +} + +function Find-Bottle { + <# + .SYNOPSIS + The newest given version carrying an Intel macOS bottle, with that + bottle's manifest digest. + #> + param( + [Parameter(Mandatory)][string]$Token, + [Parameter(Mandatory)][string[]]$Version + ) + + $headers = @{ + Authorization = "Bearer $Token" + Accept = 'application/vnd.oci.image.index.v1+json' + } + + foreach ($candidate in $Version) { + $index = Invoke-RestMethod ` + -Uri "$Registry/v2/$RegistryRepository/manifests/$candidate" ` + -Headers $headers ` + -MaximumRetryCount 3 -RetryIntervalSec 5 + + foreach ($codename in $PreferredCodenames) { + foreach ($entry in $index.manifests) { + # The index already belongs to this version, so only the + # platform component needs matching. Splitting on '.' is what + # keeps 'sonoma' from also matching 'arm64_sonoma', and rejects + # the linux and ':all' refs outright. + $ref = Get-BottleRefName -IndexEntry $entry + if ($ref -and ($ref.Split('.') -contains $codename)) { + return [pscustomobject]@{ + Version = $candidate + Digest = $entry.digest + Ref = $ref + } + } + } + } + } + + throw "No Intel macOS docker bottle in any of the $($Version.Count) published versions." +} + +function Save-BottleBlob { + <# + .SYNOPSIS + Downloads a bottle archive and verifies it against its layer digest. + #> + param( + [Parameter(Mandatory)][string]$Token, + [Parameter(Mandatory)][string]$ManifestDigest, + [Parameter(Mandatory)][string]$Path + ) + + $manifest = Invoke-RestMethod ` + -Uri "$Registry/v2/$RegistryRepository/manifests/$ManifestDigest" ` + -Headers @{ + Authorization = "Bearer $Token" + Accept = 'application/vnd.oci.image.manifest.v1+json' + } ` + -MaximumRetryCount 3 -RetryIntervalSec 5 + + $layerDigest = $manifest.layers[0].digest + + Invoke-WebRequest ` + -Uri "$Registry/v2/$RegistryRepository/blobs/$layerDigest" ` + -Headers @{ Authorization = "Bearer $Token" } ` + -MaximumRetryCount 3 -RetryIntervalSec 5 ` + -OutFile $Path + + $expected = $layerDigest -replace '^sha256:', '' + $actual = (Get-FileHash -Path $Path -Algorithm SHA256).Hash + + # -ne is case-insensitive, so Get-FileHash's uppercase output compares equal. + if ($actual -ne $expected) { + throw "Bottle digest mismatch: expected $expected, got $actual." + } +} + +#endregion Helper Functions + +$architecture = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture +if ($architecture -ne 'X64') { + throw "This installs an Intel macOS bottle, but the agent architecture is $architecture." +} + +$token = Get-RegistryToken +# Every version scanned costs a registry round-trip (~0.2s), but the whole list +# is walked rather than a fixed window: Homebrew has stopped publishing Intel +# bottles, so the newest usable version sinks further down the list over time. +$versions = @(Get-BottleVersion -Token $token) +$bottle = Find-Bottle -Token $token -Version $versions + +$archive = Join-Path ([System.IO.Path]::GetTempPath()) "docker-bottle-$([guid]::NewGuid().ToString('n')).tar.gz" + +try { + Save-BottleBlob -Token $token -ManifestDigest $bottle.Digest -Path $archive + + New-Item -ItemType Directory -Path $DestinationPath -Force | Out-Null + + # Homebrew keeps the cellar directory at the plain version even for a + # revision build, so the '29.7.2-1' bottle unpacks from 'docker/29.7.2'. + $cellarVersion = $bottle.Version -replace '-\d+$', '' + $member = "docker/$cellarVersion/bin/docker" + + # Naming the one member we want is what keeps the rest of the archive - an + # anonymous download - from ever being written to disk. + tar -xz -f $archive -C $DestinationPath --strip-components 3 $member + if ($LASTEXITCODE -ne 0) { + throw "Extracting $member from the docker bottle failed (tar exit $LASTEXITCODE)." + } +} +finally { + Remove-Item -LiteralPath $archive -Force -ErrorAction SilentlyContinue +} + +Write-Host "Installed docker $($bottle.Version) ($($bottle.Ref)) to $DestinationPath" +Write-Host "##vso[task.prependpath]$DestinationPath" diff --git a/eng/pipelines/scripts/tests/Install-DockerCli.macos.Tests.ps1 b/eng/pipelines/scripts/tests/Install-DockerCli.macos.Tests.ps1 new file mode 100644 index 0000000000..08fdcb8076 --- /dev/null +++ b/eng/pipelines/scripts/tests/Install-DockerCli.macos.Tests.ps1 @@ -0,0 +1,194 @@ +<# +.SYNOPSIS + Pester tests for Install-DockerCli.macos.ps1. + +.DESCRIPTION + These cover the bottle selection rules, which are the part that fails by + silently installing the wrong artifact rather than by crashing: picking the + newest version that is actually bottled for Intel, never taking an arm64, + linux or ':all' bottle, and reading a revision build out of the unrevised + cellar directory. + + 'Invoke-RestMethod', 'Invoke-WebRequest' and 'tar' are mocked, so the tests + need no network and no macOS. +#> + +BeforeAll { + $global:scriptPath = Join-Path $PSScriptRoot '..' 'Install-DockerCli.macos.ps1' + + # A stand-in for the bottle archive. The script verifies what it downloads + # against the digest the registry advertises, so the tests have to advertise + # this content's real hash. + $global:blobBytes = [System.Text.Encoding]::UTF8.GetBytes('not-really-a-bottle') + $blobHash = ( + [System.Security.Cryptography.SHA256]::HashData($global:blobBytes) | + ForEach-Object { $_.ToString('x2') } + ) -join '' + $global:blobDigest = "sha256:$blobHash" + + function New-Index { + <# + .SYNOPSIS + An OCI image index annotated with the given bottle ref names. + #> + param([string[]]$RefName) + + return [pscustomobject]@{ + manifests = @( + $RefName | ForEach-Object { + [pscustomobject]@{ + digest = "sha256:digest-$_" + annotations = [pscustomobject]@{ 'org.opencontainers.image.ref.name' = $_ } + } + } + ) + } + } + + function Get-TarMember { + <# + .SYNOPSIS + The archive member the script asked tar to extract. + #> + return @($global:tarArgs)[-1] + } +} + +Describe 'Install-DockerCli.macos.ps1' -Skip:([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture -ne 'X64') { + + BeforeAll { + Mock -CommandName 'Invoke-RestMethod' -MockWith { + if ($Uri -like '*/token?*') { + return [pscustomobject]@{ token = 'test-token' } + } + + if ($Uri -like '*/tags/list*') { + if ($ResponseHeadersVariable) { + Set-Variable -Name $ResponseHeadersVariable -Value @{} -Scope Global + } + return [pscustomobject]@{ tags = $global:tags } + } + + if ($Uri -like '*/manifests/sha256:*') { + $global:manifestRequests += $Uri + return [pscustomobject]@{ + layers = @([pscustomobject]@{ digest = $global:advertisedDigest }) + } + } + + $version = $Uri -replace '.*/manifests/', '' + if (-not $global:refsByVersion.ContainsKey($version)) { + throw "Unexpected manifest request for '$version'." + } + return New-Index -RefName $global:refsByVersion[$version] + } + + Mock -CommandName 'Invoke-WebRequest' -MockWith { + [System.IO.File]::WriteAllBytes($OutFile, $global:blobBytes) + } + + Mock -CommandName 'tar' -MockWith { + $global:tarArgs = $args + $global:LASTEXITCODE = 0 + } + } + + BeforeEach { + $global:tags = @('29.7.2') + $global:refsByVersion = @{ '29.7.2' = @('29.7.2.sonoma') } + $global:advertisedDigest = $global:blobDigest + $global:manifestRequests = @() + $global:tarArgs = @() + + $global:destination = Join-Path ([System.IO.Path]::GetTempPath()) "docker-cli-test-$([guid]::NewGuid().ToString('n'))" + } + + AfterEach { + if (Test-Path -LiteralPath $global:destination) { + Remove-Item -LiteralPath $global:destination -Recurse -Force -ErrorAction SilentlyContinue + } + } + + Context 'Version selection' { + + It 'Takes the newest version that is bottled for Intel' { + $global:tags = @('29.7.2', '29.8.0') + $global:refsByVersion = @{ + '29.8.0' = @('29.8.0.arm64_sequoia', '29.8.0.x86_64_linux') + '29.7.2' = @('29.7.2.sonoma') + } + + & $global:scriptPath -DestinationPath $global:destination + + Get-TarMember | Should -Be 'docker/29.7.2/bin/docker' + } + + It 'Ignores tags that are not versions' { + $global:tags = @('latest', '29.7', '29.7.2-beta', '29.7.2') + + & $global:scriptPath -DestinationPath $global:destination + + Get-TarMember | Should -Be 'docker/29.7.2/bin/docker' + } + + It 'Keeps scanning past a long run of arm64-only versions' { + # Homebrew has stopped publishing Intel bottles, so the newest usable + # version sinks further down the list with every docker release. A + # fixed scan window would eventually stop reaching it. + $global:tags = @('29.7.2') + (0..19 | ForEach-Object { "30.$_.0" }) + $global:refsByVersion = @{ '29.7.2' = @('29.7.2.sonoma') } + foreach ($i in 0..19) { + $global:refsByVersion["30.$i.0"] = @("30.$i.0.arm64_sequoia", "30.$i.0.x86_64_linux") + } + + & $global:scriptPath -DestinationPath $global:destination + + Get-TarMember | Should -Be 'docker/29.7.2/bin/docker' + } + + It 'Ranks a revision build above its base version' { + $global:tags = @('29.7.2', '29.7.2-1') + $global:refsByVersion = @{ + '29.7.2-1' = @('29.7.2.sonoma.1') + '29.7.2' = @('29.7.2.sonoma') + } + + & $global:scriptPath -DestinationPath $global:destination + + # Homebrew keeps the cellar directory at the plain version, so the + # revision must not leak into the member path. + Get-TarMember | Should -Be 'docker/29.7.2/bin/docker' + $global:manifestRequests | Should -Contain 'https://ghcr.io/v2/homebrew/core/docker/manifests/sha256:digest-29.7.2.sonoma.1' + } + } + + Context 'Platform selection' { + + It 'Prefers the newest macOS codename that is bottled' { + $global:refsByVersion = @{ '29.7.2' = @('29.7.2.ventura', '29.7.2.sonoma') } + + & $global:scriptPath -DestinationPath $global:destination + + $global:manifestRequests | Should -Contain 'https://ghcr.io/v2/homebrew/core/docker/manifests/sha256:digest-29.7.2.sonoma' + } + + It 'Never selects an arm64, linux or :all bottle' { + $global:refsByVersion = @{ + '29.7.2' = @('29.7.2.arm64_sonoma', '29.7.2.arm64_linux', '29.7.2.x86_64_linux', '29.7.2.all') + } + + { & $global:scriptPath -DestinationPath $global:destination } | + Should -Throw '*No Intel macOS docker bottle*' + } + } + + Context 'Download integrity' { + + It 'Fails when the bottle does not match the advertised digest' { + $global:advertisedDigest = 'sha256:' + ('0' * 64) + + { & $global:scriptPath -DestinationPath $global:destination } | + Should -Throw '*digest mismatch*' + } + } +} diff --git a/eng/pipelines/scripts/tests/README.md b/eng/pipelines/scripts/tests/README.md index 8906214afd..70534a9ed1 100644 --- a/eng/pipelines/scripts/tests/README.md +++ b/eng/pipelines/scripts/tests/README.md @@ -24,6 +24,7 @@ Add `-Output Detailed` to see per-test results. | File | Covers | | ---- | ------ | | `Open-LocalizationPr.Tests.ps1` | `Open-LocalizationPr.ps1` — de-duplication of the scheduled localization pull request. | +| `Install-DockerCli.macos.Tests.ps1` | `Install-DockerCli.macos.ps1` — Homebrew bottle selection for the macOS docker CLI. | -`git` and `Invoke-RestMethod` are mocked, so the tests never touch the network -or a real repository. +`git`, `tar`, `Invoke-RestMethod` and `Invoke-WebRequest` are mocked, so the +tests never touch the network or a real repository. From 849e37b4b222a15e151e28b80b22ae8e4fce5f6e Mon Sep 17 00:00:00 2001 From: priyankatiwari08 Date: Wed, 9 Sep 2026 13:14:56 +0530 Subject: [PATCH 24/51] Add bidi text preservation coverage (#4656) * Add bidi text preservation coverage Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Address directionality test review Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46c00f98-d1ca-4b28-a221-6328e7d4f1ef --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 46c00f98-d1ca-4b28-a221-6328e7d4f1ef --- .../GlobalizationTest/DirectionalityTest.cs | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/DirectionalityTest.cs diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/DirectionalityTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/DirectionalityTest.cs new file mode 100644 index 0000000000..f18cc94d06 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/DirectionalityTest.cs @@ -0,0 +1,251 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests; + +/// +/// Verifies that SqlClient preserves the logical UTF-16 representation of bidirectional text. +/// Visual direction, shaping, and mirroring are responsibilities of the consuming UI. +/// +[Trait("Set", "3")] +public sealed class DirectionalityTest +{ + private static readonly string[] s_bidiText = + { + "\u0645\u0631\u062D\u0628\u0627 Microsoft 01 - \u0639\u0627\u0644\u0645 \U0001F310", + "\u05E9\u05DC\u05D5\u05DD Microsoft 01 - \u05E2\u05D5\u05DC\u05DD \U0001F310" + }; + + /// + /// Ensures mixed Arabic/Hebrew, Latin, numeric, punctuation, and supplementary characters + /// round-trip unchanged through parameters, readers, sequential streaming, and bulk copy. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] + [InlineData(false)] + [InlineData(true)] + public async Task BidiText_RoundTripsWithoutTransformation(bool async) + { + using SqlConnection setupConnection = new(DataTestUtility.TCPConnectionString); + await OpenConnection(setupConnection, async); + + using Table sourceTable = new( + setupConnection, + "DirectionalitySource", + "(Id int NOT NULL, Value nvarchar(max) NOT NULL)"); + using Table destinationTable = new( + setupConnection, + "DirectionalityDestination", + "(Id int NOT NULL, Value nvarchar(max) NOT NULL)"); + + await InsertValues(setupConnection, sourceTable.Name, async); + await VerifyOrdinaryReader(setupConnection, sourceTable.Name, async); + await VerifyGetChars(setupConnection, sourceTable.Name, async); + await VerifyTextReader(setupConnection, sourceTable.Name, async); + await CopyValues(sourceTable.Name, destinationTable.Name, async); + await VerifyOrdinaryReader(setupConnection, destinationTable.Name, async); + } + + /// + /// Opens a connection through the requested synchronous or asynchronous API. + /// + /// The connection to open. + /// Whether to use the asynchronous API. + /// A task representing the open operation. + private static async Task OpenConnection(SqlConnection connection, bool async) + { + if (async) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + } + + /// + /// Inserts the bidi samples through explicitly typed Unicode parameters. + /// + /// The open connection used to insert the samples. + /// The table that receives the samples. + /// Whether to use asynchronous command execution. + /// A task representing the insert operations. + private static async Task InsertValues(SqlConnection connection, string tableName, bool async) + { + using SqlCommand command = new($"INSERT INTO {tableName} (Id, Value) VALUES (@id, @value)", connection); + SqlParameter idParameter = command.Parameters.Add("@id", SqlDbType.Int); + SqlParameter valueParameter = command.Parameters.Add("@value", SqlDbType.NVarChar, -1); + + for (int index = 0; index < s_bidiText.Length; index++) + { + idParameter.Value = index; + valueParameter.Value = s_bidiText[index]; + + if (async) + { + await command.ExecuteNonQueryAsync(); + } + else + { + command.ExecuteNonQuery(); + } + } + } + + /// + /// Reads complete strings through ordinary reader accessors and compares their UTF-16 content. + /// + /// The open connection used to read the samples. + /// The table containing the samples. + /// Whether to use asynchronous reader APIs. + /// A task representing the verification operation. + private static async Task VerifyOrdinaryReader(SqlConnection connection, string tableName, bool async) + { + using SqlCommand command = new($"SELECT Id, Value FROM {tableName} ORDER BY Id", connection); + using SqlDataReader reader = async + ? await command.ExecuteReaderAsync() + : command.ExecuteReader(); + + for (int index = 0; index < s_bidiText.Length; index++) + { + bool hasRow = async ? await reader.ReadAsync() : reader.Read(); + Assert.True(hasRow); + Assert.Equal(index, reader.GetInt32(0)); + AssertOrdinalEqual(s_bidiText[index], reader.GetString(1)); + string fieldValue = async + ? await reader.GetFieldValueAsync(1) + : reader.GetFieldValue(1); + AssertOrdinalEqual(s_bidiText[index], fieldValue); + } + + Assert.False(async ? await reader.ReadAsync() : reader.Read()); + } + + /// + /// Reads one UTF-16 code unit at a time to cover direction and surrogate boundaries in GetChars. + /// + /// The open connection used to read the samples. + /// The table containing the samples. + /// Whether to use asynchronous reader execution. + /// A task representing the verification operation. + private static async Task VerifyGetChars(SqlConnection connection, string tableName, bool async) + { + using SqlCommand command = new($"SELECT Value FROM {tableName} ORDER BY Id", connection); + using SqlDataReader reader = async + ? await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : command.ExecuteReader(CommandBehavior.SequentialAccess); + + for (int index = 0; index < s_bidiText.Length; index++) + { + bool hasRow = async ? await reader.ReadAsync() : reader.Read(); + Assert.True(hasRow); + + StringBuilder result = new(); + char[] buffer = new char[1]; + long dataIndex = 0; + long charsRead; + do + { + charsRead = reader.GetChars(0, dataIndex, buffer, 0, buffer.Length); + result.Append(buffer, 0, (int)charsRead); + dataIndex += charsRead; + } + while (charsRead != 0); + + AssertOrdinalEqual(s_bidiText[index], result.ToString()); + } + } + + /// + /// Reads bidi text through the sequential TextReader using small sync or async buffer operations. + /// + /// The open connection used to read the samples. + /// The table containing the samples. + /// Whether to use asynchronous reader and text operations. + /// A task representing the verification operation. + private static async Task VerifyTextReader(SqlConnection connection, string tableName, bool async) + { + using SqlCommand command = new($"SELECT Value FROM {tableName} ORDER BY Id", connection); + using SqlDataReader reader = async + ? await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : command.ExecuteReader(CommandBehavior.SequentialAccess); + + for (int index = 0; index < s_bidiText.Length; index++) + { + bool hasRow = async ? await reader.ReadAsync() : reader.Read(); + Assert.True(hasRow); + + using TextReader textReader = reader.GetTextReader(0); + StringBuilder result = new(); + char[] buffer = new char[2]; + int charsRead; + do + { + charsRead = async + ? await textReader.ReadAsync(buffer, 0, buffer.Length) + : textReader.Read(buffer, 0, buffer.Length); + result.Append(buffer, 0, charsRead); + } + while (charsRead != 0); + + AssertOrdinalEqual(s_bidiText[index], result.ToString()); + } + } + + /// + /// Copies the Unicode rows through streaming SqlBulkCopy using the requested execution mode. + /// + /// The table containing the source rows. + /// The table receiving the copied rows. + /// Whether to use asynchronous reader and bulk-copy APIs. + /// A task representing the copy operation. + private static async Task CopyValues(string sourceTableName, string destinationTableName, bool async) + { + using SqlConnection sourceConnection = new(DataTestUtility.TCPConnectionString); + using SqlConnection destinationConnection = new(DataTestUtility.TCPConnectionString); + await OpenConnection(sourceConnection, async); + await OpenConnection(destinationConnection, async); + + using SqlCommand command = new($"SELECT Id, Value FROM {sourceTableName} ORDER BY Id", sourceConnection); + using SqlDataReader reader = async + ? await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : command.ExecuteReader(CommandBehavior.SequentialAccess); + using SqlBulkCopy bulkCopy = new(destinationConnection) + { + DestinationTableName = destinationTableName, + EnableStreaming = true + }; + bulkCopy.ColumnMappings.Add(0, 0); + bulkCopy.ColumnMappings.Add(1, 1); + + if (async) + { + await bulkCopy.WriteToServerAsync(reader); + } + else + { + bulkCopy.WriteToServer(reader); + } + } + + /// + /// Compares strings ordinally so the assertion checks logical storage rather than visual rendering. + /// + /// The original UTF-16 string. + /// The round-tripped UTF-16 string. + private static void AssertOrdinalEqual(string expected, string actual) + { + Assert.True( + string.Equals(expected, actual, StringComparison.Ordinal), + $"Expected and actual UTF-16 values differ. Expected length: {expected.Length}; actual length: {actual?.Length}."); + } +} From a869e6db0951b480ad6b7f6ed6d2fdee79f228dd Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:43:45 -0300 Subject: [PATCH 25/51] Disable legacy package-reference pipelines on main (#4647) --- .../dotnet-sqlclient-ci-package-reference-pipeline.yml | 4 +++- eng/pipelines/sqlclient-pr-package-ref-pipeline.yml | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml index 74fa022a6c..a1b5f14216 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml @@ -59,7 +59,9 @@ trigger: branches: include: # GitHub main and release branches. - - main + # + # GOTCHA: Currently disabled on main due to limited resources. + #- main - release/* # ADO main and release branches. diff --git a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml index 7623f83ce4..f4427f82e8 100644 --- a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml @@ -34,9 +34,11 @@ pr: branches: include: # GitHub repo branch targets that will trigger PR validation builds. - - dev/* - - feat/* - - main + # + # GOTCHA: Currently disabled on all but release/* due to limited resources. + #- dev/* + #- feat/* + #- main - release/* paths: From 87d48199332f83fa0a7ea72a6e277d9edb717e3b Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:48:02 -0300 Subject: [PATCH 26/51] Pipelines | Pre-compute all OneBranch package and file versions (#4652) * Pipelines | Pre-compute all OneBranch package and file versions Make the compute-versions stage the single source of every version the OneBranch build jobs consume, so nothing is re-derived downstream. - Remove the addRevision mode entirely. Package versions now have a single shape driven by the pipeline build number, and the 16-bit revision wrapping, the four-part package base handling, and the Build.BuildId plumbing are gone. - Move package version stamping out of PowerShell and into Versions.props. BuildSuffix now does what it always documented: it turns a stable base into a prerelease. Any version carrying a prerelease tag, from either source, is stamped with the build number; released versions are left untouched. - Publish SqlClient and SqlServer file versions from the compute-versions stage and pass them into the build jobs, which previously received a raw build number and re-derived the file version through MSBuild. build.proj gains opt-in FileVersion* arguments, so PR/CI and local builds are unchanged. - Fix SBOM metadata, which reported the pipeline run number as the version of a single hardcoded package name. Each build job now supplies the name and computed version of the package it produces, and jobs that publish no packages disable SBOM generation instead. * Add version composition target tests for PR 4652 * Fix version extraction example for PR 4652 * Validate computed file version component count for PR 4652 * Require four-part numeric file versions for PR 4652 * Disable SBOM generation in non-producing jobs for PR 4652 --- .../onebranch-pipeline-design.instructions.md | 1 + ...sqlclient-package-versions.instructions.md | 56 +-- build.proj | 83 +++- .../onebranch/jobs/build-buildproj-job.yml | 18 +- .../jobs/publish-nuget-package-job.yml | 5 + .../onebranch/jobs/publish-symbols-job.yml | 3 + .../jobs/validate-signed-package-job.yml | 4 + .../onebranch/scripts/compute-versions.ps1 | 229 +++-------- .../onebranch/scripts/tests/README.md | 5 +- .../scripts/tests/compute-versions.Tests.ps1 | 354 ++++++++++++++---- .../onebranch/sqlclient-non-official.yml | 16 +- .../onebranch/sqlclient-official.yml | 15 +- .../onebranch/stages/build-stages.yml | 30 +- .../stages/compute-versions-stage.yml | 20 +- .../onebranch/steps/build-buildproj-step.yml | 6 +- .../onebranch/steps/pack-buildproj-step.yml | 6 +- .../steps/roslyn-analyzers-buildproj-step.yml | 9 +- .../onebranch/variables/package-variables.yml | 17 - src/Microsoft.Data.SqlClient/Versions.props | 37 +- src/Microsoft.SqlServer.Server/Versions.props | 36 +- 20 files changed, 529 insertions(+), 421 deletions(-) diff --git a/.github/instructions/onebranch-pipeline-design.instructions.md b/.github/instructions/onebranch-pipeline-design.instructions.md index 272369dfb5..7a24fd3799 100644 --- a/.github/instructions/onebranch-pipeline-design.instructions.md +++ b/.github/instructions/onebranch-pipeline-design.instructions.md @@ -135,6 +135,7 @@ Variable groups: - Jobs that produce no assemblies (symbol publishing, signed-package validation, version computation) set `ob_sdl_apiscan_enabled: false` rather than reporting a name/version - Each build job also sets `ob_sdl_apiscan_softwareFolder` and `ob_sdl_apiscan_symbolsFolder` to its per-package `apiScan//dlls` and `apiScan//pdbs` paths - CodeQL, SBOM, Policheck (`break: true`): enabled in both pipelines +- SBOM package name/version are resolvable **only** from the pipeline's `globalSdl.sbom` block — OneBranch's artifact-publishing path reads `globalSdl.sbom.packageName`/`packageVersion` directly and has no per-job equivalent (the `templateContext.sdl.sbom` override only applies to the native 1ES Stages entry point, which this repo does not use). Because the pipeline produces six differently-named and independently-versioned packages, `globalSdl.sbom` indirects through the `$(sbomPackageName)` / `$(sbomPackageVersion)` variables, which each build job sets to its own `packageFullName` and computed `packageVersion`. Jobs that publish no packages (version computation, symbol publishing) set `ob_sdl_sbom_enabled: false` alongside their existing APIScan/BinSkim opt-outs, so the variables never need pipeline-level defaults - asyncSdl `enabled: false` in both; individual sub-tools (CredScan, BinSkim, Armory, Roslyn) configured underneath - Policheck exclusions: `$(REPO_ROOT)\.config\PolicheckExclusions.xml` - CredScan suppressions: `$(REPO_ROOT)/.config/CredScanSuppressions.json` diff --git a/.github/instructions/sqlclient-package-versions.instructions.md b/.github/instructions/sqlclient-package-versions.instructions.md index 1692082a68..691367d9de 100644 --- a/.github/instructions/sqlclient-package-versions.instructions.md +++ b/.github/instructions/sqlclient-package-versions.instructions.md @@ -40,7 +40,7 @@ Each `Versions.props` uses a 3-tier `` block: | Priority | Condition | PackageVersion | FileVersion | |----------|-----------|----------------|-------------| | 1 | `PackageVersion` explicitly provided | Used as-is | Strip prerelease + append BuildNumber | -| 2 | `BuildNumber` provided (non-zero) | `NextVersion[-BuildSuffix+BuildNumber]` | `NextVersion.Split('-')[0].BuildNumber` | +| 2 | `BuildNumber` provided (non-zero) | `NextVersion[-BuildSuffix]`, then `.BuildNumber` appended if that carries a prerelease tag | `NextVersion.Split('-')[0].BuildNumber` | | 3 | Nothing provided | `NextVersion-dev` | `NextVersion.Split('-')[0].0` | For every family package, `` is `SqlClient` (e.g. `-p:SqlClientPackageVersion=...`); for @@ -79,7 +79,7 @@ Microsoft.SqlServer.Server it is `SqlServer`. - Versions computed in `compute-versions-ci-stage.yml` (runs `GetVersions*` targets with `-p:BuildSuffix=pr -p:BuildNumber=...`) - Falls into Priority 2 with BuildSuffix present. -- **Result:** `7.1.0-preview1-pr15401` / FileVersion `7.1.0.15401` +- **Result:** `7.1.0-preview1-pr.15401` / FileVersion `7.1.0.15401` - Dependencies are project references — all packages built together in-tree. **Mode:** Package (PR package-ref validation) @@ -93,7 +93,7 @@ Microsoft.SqlServer.Server it is `SqlServer`. Same structure as PR but passes `buildSuffix: 'ci'` explicitly. -- **Result:** `7.1.0-preview1-ci15401` / FileVersion `7.1.0.15401` +- **Result:** `7.1.0-preview1-ci.15401` / FileVersion `7.1.0.15401` ### OneBranch Pipeline (official) @@ -104,8 +104,8 @@ Uses the full `compute-versions-stage.yml` machinery: #### Step A: Compute Versions (dedicated early stage) 1. Runs the `GetVersionsSqlClient` and `GetVersionsSqlServer` MSBuild targets against `build.proj`. -2. Each target calls `dotnet build -getProperty:PackageVersion` with `BuildNumber` but **no BuildSuffix**. -3. Falls into Priority 2 without BuildSuffix → `PackageVersion = NextVersion` as-is (e.g. `7.1.0-preview1`). +2. Each target calls `dotnet build -getProperty:PackageVersion` and `-getProperty:FileVersion` with `BuildNumber` but **no BuildSuffix**. +3. Falls into Priority 2 without BuildSuffix. `NextVersion` already carries a prerelease tag on `main`, so the build number is appended (e.g. `7.1.0-preview1.26238.3`); on a release branch the stable `NextVersion` is used as-is (e.g. `7.1.0`). 4. `GetVersionsSqlServer` also extracts `SqlServerPublishedVersion` (the SqlClient family has no published version). #### Step B: Resolve Effective Versions @@ -136,15 +136,12 @@ Each downstream build job receives: Since an explicit `PackageVersion` is provided, Versions.props hits Priority 1 — uses the value verbatim. -#### Package Version Shapes: `addRevision` +#### Package Version Shapes -Both OneBranch pipelines expose `addRevision` (default `false`), which selects between two mutually -exclusive package version shapes. Their human-readable run name is `$(Year:YY)$(DayOfYear)$(Rev:.r)`, -and the compute stage receives both that run name (as `Build.BuildNumber`) and the globally unique -`Build.BuildId` (as the revision). - -**Default path — `addRevision: false`.** The pipeline run name is appended after any prerelease -suffix, reproducing the shape shipped by earlier previews. `Build.BuildId` is not used: +Both OneBranch pipelines use the human-readable run name `$(Year:YY)$(DayOfYear)$(Rev:.r)`, which the +compute stage receives as `Build.BuildNumber`. That run name drives the single supported package +version shape: it is appended after any prerelease suffix, reproducing the shape shipped by earlier +previews. - `1.2.3` stays `1.2.3` — non-preview releases are never stamped with a build number - `1.2.3-preview1` becomes `1.2.3-preview1.`, e.g. `7.1.0-preview3.26238.3` @@ -152,34 +149,15 @@ suffix, reproducing the shape shipped by earlier previews. `Build.BuildId` is no Note the asymmetry: the *package* version omits the build number for non-preview releases, but the *file* version always carries one in its fourth component. This keeps every shipped assembly -date-encoded and traceable to the run that produced it, while preserving the released package -version customers expect. - -**Opt-in path — `addRevision: true`.** Version revisions come from `Build.BuildId` instead, which is -mapped into the unsigned 16-bit file-version range before canonical file versions are evaluated: - -```text -revision = ((Build.BuildId - 1) % 65535) + 1 -``` - -Build IDs `1` through `65535` map directly; subsequent IDs wrap back through that range. The compute -stage logs the mapping whenever wrapping occurs because the revision can then collide with an earlier -run. The mapped value is inserted before any package prerelease suffix so package and file versions -use the same revision: - -- `1.2.3` becomes `1.2.3.` -- `1.2.3-preview1` becomes `1.2.3.-preview1` -- The matching file version is `1.2.3.` - -This shape exists for repeated test publishes of the same base version, where each run needs a -distinct package version. `Build.BuildNumber` is not used on this path. +date-encoded, while preserving the released package version customers expect. -Only packages built in the current run are revised or stamped. When `buildSqlServer` is `false`, the -effective SqlServer version remains `SqlServerPublishedVersion` so dependency restore continues to -request the package that actually exists on NuGet. +The file version's fourth component is only the *date* segment of the run name, because a four-part +file version cannot hold the full `.` value. Repeated runs on the same day therefore share +a file version even though their package versions differ. -An explicit four-part package version is also treated as the complete file version base by both -canonical Versions.props files; they do not append `FileVersionBuildNumber` as a fifth component. +Only packages built in the current run are stamped. When `buildSqlServer` is `false`, the effective +SqlServer version remains `SqlServerPublishedVersion` so dependency restore continues to request the +package that actually exists on NuGet. #### Summary diff --git a/build.proj b/build.proj index 31795fc6a7..2e19233b2a 100644 --- a/build.proj +++ b/build.proj @@ -96,6 +96,21 @@ packages — use PackageVersionSqlClient (below) to set the version for the entire family. --> + + + + -p:SqlClientNextVersion=$(SqlClientNextVersion) + + + + + -p:SqlClientFileVersion=$(FileVersionSqlClient) + + + + + + -p:SqlServerNextVersion=$(SqlServerNextVersion) + + + + + -p:SqlServerFileVersion=$(FileVersionSqlServer) + + - <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlClientNextVersionArgument) <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) - <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlClientNextVersionArgument) <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) @@ -476,7 +535,7 @@ - <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlServerNextVersionArgument) <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) @@ -486,7 +545,7 @@ - <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlServerNextVersionArgument) <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) @@ -574,6 +633,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -605,6 +665,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -636,6 +697,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -679,6 +741,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -839,6 +902,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -866,6 +930,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -946,6 +1011,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -972,6 +1038,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -1051,6 +1118,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -1077,6 +1145,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -1153,6 +1222,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) @@ -1176,6 +1246,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) @@ -1219,6 +1290,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlServerArgument) + $(FileVersionSqlServerArgument) $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) @@ -1242,6 +1314,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlServerArgument) + $(FileVersionSqlServerArgument) $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) diff --git a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml index f750ed4b91..824776575d 100644 --- a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml +++ b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml @@ -66,8 +66,9 @@ parameters: type: object default: [] - # Positive unsigned 16-bit revision used by build.proj to derive assembly file versions. - - name: revision + # Assembly file version to stamp (required). Pre-computed by the compute-versions stage so no + # build job re-derives it. The assembly version is derived from this by Versions.props. + - name: fileVersion type: string # The full name of the package. This is used in the job name, and to form DLL and PDB filenames @@ -126,6 +127,13 @@ jobs: ob_sdl_apiscan_softwareName: ${{ parameters.packageFullName }} ob_sdl_apiscan_versionNumber: ${{ parameters.apiScanSoftwareVersion }} + # SBOM identity for this job's artifact. OneBranch reads the SBOM package name/version only + # from the pipeline's globalSdl block, which has no per-job form, so that block indirects + # through these variables and each build job supplies its own values. Jobs that publish no + # packages set ob_sdl_sbom_enabled to false instead. + sbomPackageName: ${{ parameters.packageFullName }} + sbomPackageVersion: ${{ parameters.packageVersion }} + steps: - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self @@ -166,7 +174,7 @@ jobs: parameters: dependencyArguments: $(sqlServerVersionArgument) packageShortName: ${{ parameters.packageShortName }} - revision: ${{ parameters.revision }} + fileVersion: ${{ parameters.fileVersion }} versionPropertySuffix: ${{ parameters.versionPropertySuffix }} packageVersion: ${{ parameters.packageVersion }} @@ -176,7 +184,7 @@ jobs: buildConfiguration: Release dependencyArguments: $(sqlServerVersionArgument) packageShortName: ${{ parameters.packageShortName }} - revision: ${{ parameters.revision }} + fileVersion: ${{ parameters.fileVersion }} versionPropertySuffix: ${{ parameters.versionPropertySuffix }} packageVersion: ${{ parameters.packageVersion }} @@ -224,7 +232,7 @@ jobs: dependencyArguments: $(sqlServerVersionArgument) packageFullName: ${{ parameters.packageFullName }} packageShortName: ${{ parameters.packageShortName }} - revision: ${{ parameters.revision }} + fileVersion: ${{ parameters.fileVersion }} versionPropertySuffix: ${{ parameters.versionPropertySuffix }} packageVersion: ${{ parameters.packageVersion }} diff --git a/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml b/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml index cf7dd6a56b..e63c72bf58 100644 --- a/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml +++ b/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml @@ -74,6 +74,11 @@ jobs: - name: ob_outputDirectory value: $(JOB_OUTPUT) + # This job republishes an already-built package, whose SBOM came from its build job. It sets + # no sbomPackage* values, so leaving SBOM enabled would emit one with unresolved macros. + - name: ob_sdl_sbom_enabled + value: false + - name: artifactPath value: $(Pipeline.Workspace)/${{ parameters.artifactName }} diff --git a/eng/pipelines/onebranch/jobs/publish-symbols-job.yml b/eng/pipelines/onebranch/jobs/publish-symbols-job.yml index af59e54fc2..ab45b5f4c8 100644 --- a/eng/pipelines/onebranch/jobs/publish-symbols-job.yml +++ b/eng/pipelines/onebranch/jobs/publish-symbols-job.yml @@ -64,6 +64,9 @@ jobs: value: false - name: ob_sdl_binskim_enabled value: false + # No packages are published here, so there is nothing to describe in an SBOM. + - name: ob_sdl_sbom_enabled + value: false # Path where the downloaded artifact will be placed. - name: artifactPath value: '$(Pipeline.Workspace)/${{ parameters.packageFullName }}' diff --git a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml b/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml index 476bb5de30..818b902cf7 100644 --- a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml +++ b/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml @@ -53,6 +53,10 @@ jobs: - name: ob_sdl_apiscan_enabled value: false + # Likewise it produces no package, and sets no sbomPackage* values for globalSdl to resolve. + - name: ob_sdl_sbom_enabled + value: false + # Path within the downloaded artifact where NuGet packages are located. - name: artifactPath value: '$(Pipeline.Workspace)\${{ parameters.artifactName }}' diff --git a/eng/pipelines/onebranch/scripts/compute-versions.ps1 b/eng/pipelines/onebranch/scripts/compute-versions.ps1 index 93e8d318aa..4bce0b1495 100644 --- a/eng/pipelines/onebranch/scripts/compute-versions.ps1 +++ b/eng/pipelines/onebranch/scripts/compute-versions.ps1 @@ -12,63 +12,39 @@ The published SqlServer version is needed when SqlServer is not built because downstream SqlClient projects restore that existing package from NuGet. - Two mutually exclusive package version shapes are supported: - - When AddRevision is true, BuildNumber is ignored if specified. For non-preview releases, versions - will be 1.2.3.. For preview releases, versions will be 1.2.3.-previewX. - - When AddRevision is false, BuildNumber must be specified. Non-preview versions will be 1.2.3, and - preview releases will be 1.2.3-previewX.. - - This asymmetry maintains the existing package versioning practice where previews include a - date-coded build number, but normal releases do not. + Package versions take a single shape, produced by Versions.props from the pipeline build number. + Preview versions carry the full build number after the prerelease suffix, such as + 1.2.3-preview1.26238.3. Stable versions are left exactly as declared in Versions.props, such as + 1.2.3, because released packages are not stamped with a build number. File versions are always four-part and always carry a build number in the fourth component, even - when the package version does not: - - - AddRevision true. The fourth component is the mapped revision, so package and file versions - agree. A package version of 1.2.3.34430-preview1 has file version 1.2.3.34430. - - - AddRevision false. The fourth component is the first segment of BuildNumber. A package - version of 1.2.3-preview1.26238.3 has file version 1.2.3.26238, and a stable package version - of 1.2.3 still has file version 1.2.3.26238. This keeps every file version date-encoded and - traceable back to the run that produced it. + when the package version does not. Versions.props derives that component from the date segment of + BuildNumber, so a package version of 1.2.3-preview1.26238.3 has file version 1.2.3.26238, and a + stable package version of 1.2.3 still has file version 1.2.3.26238. That segment is date-coded, + so repeated runs on the same day share a file version even though their package versions differ. - The supplied revision is mapped to the range 1 through 65535 before canonical versions - are evaluated so every file version has a valid fourth component. Revisions above 65535 wrap - through the valid unsigned 16-bit file-version range. When AddRevision is true the script logs - the mapping because the revision may then collide with an earlier run. An unbuilt SqlServer - package is never revised or stamped with a build number because its effective version must - continue to identify the package that already exists on NuGet. + An unbuilt SqlServer package is never stamped with a build number because its effective version + must continue to identify the package that already exists on NuGet. The script emits these output variables for downstream stages: - - VersionRevision - SqlClientPackageVersion + - SqlClientFileVersion - SqlServerPackageVersion + - SqlServerFileVersion - SqlClientApiScanVersion - SqlServerApiScanVersion .PARAMETER ProjectPath Absolute or relative path to the repository build.proj file. -.PARAMETER Revision - Positive integer used to distinguish versions. Values above 65535 are wrapped into the unsigned - 16-bit revision range. Only consumed when AddRevision is true. - .PARAMETER BuildNumber - Pipeline build number in the form ., such as 26238.3. Required when AddRevision is - false, and ignored when AddRevision is true. When required, it is appended to prerelease package - versions and its first segment becomes the file-version build number, so file versions remain - date-encoded. + Pipeline build number in the form ., such as 26238.3. Versions.props appends it to + prerelease package versions and derives the file-version build number from its date segment. .PARAMETER BuildSqlServer Whether this run builds Microsoft.SqlServer.Server. When false, the effective SqlServer package version is its last published version and its file version is not consumed downstream. -.PARAMETER AddRevision - Whether to insert the revision into package versions built during this run. - Defaults to false in both top-level OneBranch pipelines. - .PARAMETER DotnetPath dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter primarily supports isolated testing and specialized agent configurations. @@ -76,33 +52,19 @@ .EXAMPLE ./compute-versions.ps1 ` -ProjectPath ./build.proj ` - -Revision 165500 ` -BuildNumber 26238.3 ` - -BuildSqlServer $true ` - -AddRevision $false + -BuildSqlServer $true - Computes versions for an official-style run that builds SqlServer. Prerelease package versions - become 7.1.0-preview3.26238.3 and file versions become 7.1.0.26238. + Computes versions for a run that builds SqlServer. Prerelease package versions become + 7.1.0-preview3.26238.3 and file versions become 7.1.0.26238. .EXAMPLE ./compute-versions.ps1 ` -ProjectPath ./build.proj ` - -Revision 165500 ` - -BuildSqlServer $true ` - -AddRevision $true - - Computes versions for a run that builds SqlServer and appends mapped revision 34430 to both - package families and their file versions. BuildNumber is not supplied because AddRevision is - true. - -.EXAMPLE - ./compute-versions.ps1 ` - -ProjectPath ./build.proj ` - -Revision 165500 ` - -BuildSqlServer $false ` - -AddRevision $true + -BuildNumber 26238.3 ` + -BuildSqlServer $false - Revises the SqlClient family versions while retaining SqlServerPublishedVersion for dependency + Stamps the SqlClient family versions while retaining SqlServerPublishedVersion for dependency restore because SqlServer is not built in this run. .NOTES @@ -117,20 +79,13 @@ param( [ValidateScript({ Test-Path -LiteralPath $_ -PathType Leaf })] [string]$ProjectPath, - [Parameter(Mandatory = $true, HelpMessage = "Positive integer version revision.")] - [ValidateRange(1, [long]::MaxValue)] - [long]$Revision, - - [Parameter(HelpMessage = "Pipeline build number, such as 26238.3. Required when AddRevision is false.")] - [ValidatePattern("^$|^\d+\.\d+$")] - [string]$BuildNumber = "", + [Parameter(Mandatory = $true, HelpMessage = "Pipeline build number, such as 26238.3.")] + [ValidatePattern("^\d+\.\d+$")] + [string]$BuildNumber, [Parameter(Mandatory = $true, HelpMessage = "Whether Microsoft.SqlServer.Server is built in this run.")] [bool]$BuildSqlServer, - [Parameter(Mandatory = $true, HelpMessage = "Whether to append the revision to built package versions.")] - [bool]$AddRevision, - [Parameter(HelpMessage = "dotnet executable to invoke.")] [ValidateNotNullOrEmpty()] [string]$DotnetPath = "dotnet" @@ -139,11 +94,6 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" -$wrappedRevision = (($Revision - 1) % 65535) + 1 -if ($AddRevision -and $Revision -gt 65535) { - Write-Host "Revision $Revision exceeds the unsigned 16-bit limit and wrapped to $wrappedRevision; this revision may collide with an earlier run." -} - <# .SYNOPSIS Extracts the first value associated with a labeled GetVersions target output line. @@ -179,7 +129,7 @@ function Get-LabeledValue { GetVersions target suffix: SqlClient or SqlServer. .OUTPUTS - An object containing PackageVersion and PublishedVersion. + An object containing PackageVersion, FileVersion, and PublishedVersion. #> function Get-CanonicalVersions { param( @@ -188,93 +138,28 @@ function Get-CanonicalVersions { ) $output = & $DotnetPath build $ProjectPath ` - -t:"GetVersions${Label}" -v:m -nologo -p:BuildNumber=$wrappedRevision 2>&1 + -t:"GetVersions${Label}" -v:m -nologo -p:BuildNumber=$BuildNumber 2>&1 if ($LASTEXITCODE -ne 0) { throw ($output -join [Environment]::NewLine) } $packageVersion = Get-LabeledValue -Output $output -Label "PackageVersion" + $fileVersion = Get-LabeledValue -Output $output -Label "FileVersion" $publishedVersion = Get-LabeledValue -Output $output -Label "PublishedVersion" if ([string]::IsNullOrWhiteSpace($packageVersion)) { throw "Failed to extract PackageVersion for ${Label}.`n$($output -join [Environment]::NewLine)" } + if ([string]::IsNullOrWhiteSpace($fileVersion)) { + throw "Failed to extract FileVersion for ${Label}.`n$($output -join [Environment]::NewLine)" + } [pscustomobject]@{ PackageVersion = $packageVersion + FileVersion = $fileVersion PublishedVersion = $publishedVersion } } -<# -.SYNOPSIS - Inserts a numeric revision before a package version's prerelease suffix. - -.PARAMETER Version - Package version with a three-part numeric base and optional prerelease suffix. - -.PARAMETER Revision - Revision in the unsigned 16-bit file-version range. - -.OUTPUTS - A four-part package version preserving the original prerelease suffix. -#> -function Add-VersionRevision { - param( - [string]$Version, - [ValidateRange(1, 65535)] - [int]$Revision - ) - - $parts = $Version -split "-", 2 - if ($parts[0] -notmatch "^\d+\.\d+\.\d+$") { - throw "Expected a three-part numeric version base, but received '$Version'." - } - - $versionWithRevision = "$($parts[0]).$Revision" - if ($parts.Count -eq 2) { - return "$versionWithRevision-$($parts[1])" - } - - return $versionWithRevision -} - -<# -.SYNOPSIS - Appends a pipeline build number to a prerelease package version. - -.DESCRIPTION - Reproduces the version shape used by earlier previews, where the build number follows the - prerelease suffix, such as 7.1.0-preview3.26238.3. Stable versions are returned unchanged - because released packages are not stamped with a build number. - -.PARAMETER Version - Package version with a three-part numeric base and optional prerelease suffix. - -.PARAMETER BuildNumber - Pipeline build number to append. - -.OUTPUTS - The package version with the build number appended to its prerelease suffix, or the original - version when it carries no prerelease suffix. -#> -function Add-VersionBuildNumber { - param( - [string]$Version, - [string]$BuildNumber - ) - - $parts = $Version -split "-", 2 - if ($parts[0] -notmatch "^\d+\.\d+\.\d+$") { - throw "Expected a three-part numeric version base, but received '$Version'." - } - - if ($parts.Count -eq 2) { - return "$($parts[0])-$($parts[1]).$BuildNumber" - } - - return $Version -} - <# .SYNOPSIS Extracts the major.minor components from a package version. @@ -316,14 +201,18 @@ function Set-PipelineOutputVariable { Write-Host "##vso[task.setvariable variable=${Name};isOutput=true]$Value" } -Write-Host "Extracting versions with revision=$Revision (wrapped=$wrappedRevision)..." +Write-Host "Extracting versions with build number $BuildNumber..." $sqlClientVersions = Get-CanonicalVersions -Label "SqlClient" $sqlServerVersions = Get-CanonicalVersions -Label "SqlServer" -Write-Host " SqlClient: pkg=$($sqlClientVersions.PackageVersion)" -Write-Host " SqlServer: pkg=$($sqlServerVersions.PackageVersion) pub=$($sqlServerVersions.PublishedVersion)" +Write-Host " SqlClient: pkg=$($sqlClientVersions.PackageVersion) file=$($sqlClientVersions.FileVersion)" +Write-Host " SqlServer: pkg=$($sqlServerVersions.PackageVersion) file=$($sqlServerVersions.FileVersion) pub=$($sqlServerVersions.PublishedVersion)" $sqlClientPackageVersion = $sqlClientVersions.PackageVersion +$sqlClientFileVersion = $sqlClientVersions.FileVersion + +# An unbuilt SqlServer resolves to its published version, which no build job stamps, so it has no +# effective file version. $sqlServerPackageVersion = if ($BuildSqlServer) { $sqlServerVersions.PackageVersion } else { @@ -332,46 +221,11 @@ $sqlServerPackageVersion = if ($BuildSqlServer) { } $sqlServerVersions.PublishedVersion } - -$fileVersionBuildNumber = $null - -if ($AddRevision) { - $sqlClientPackageVersion = Add-VersionRevision ` - -Version $sqlClientPackageVersion ` - -Revision $wrappedRevision - if ($BuildSqlServer) { - $sqlServerPackageVersion = Add-VersionRevision ` - -Version $sqlServerPackageVersion ` - -Revision $wrappedRevision - } - - # The revision is already the fourth component of the package version, so it is also the - # file-version build number. - $fileVersionBuildNumber = $wrappedRevision - Write-Host "Version revision: $wrappedRevision (input=$Revision)" -} -else { - if ([string]::IsNullOrWhiteSpace($BuildNumber)) { - throw "BuildNumber is required when AddRevision is false." - } - - $fileVersionBuildNumber = $BuildNumber.Split(".")[0] - - $sqlClientPackageVersion = Add-VersionBuildNumber ` - -Version $sqlClientPackageVersion ` - -BuildNumber $BuildNumber - if ($BuildSqlServer) { - $sqlServerPackageVersion = Add-VersionBuildNumber ` - -Version $sqlServerPackageVersion ` - -BuildNumber $BuildNumber - } - - Write-Host "Version build number: $BuildNumber (file version build number=$fileVersionBuildNumber)" -} +$sqlServerFileVersion = if ($BuildSqlServer) { $sqlServerVersions.FileVersion } else { "" } Write-Host "Effective versions:" -Write-Host " SqlClient (family): $sqlClientPackageVersion" -Write-Host " SqlServer: $sqlServerPackageVersion" +Write-Host " SqlClient (family): $sqlClientPackageVersion (file $sqlClientFileVersion)" +Write-Host " SqlServer: $sqlServerPackageVersion (file $sqlServerFileVersion)" $sqlClientApiScanVersion = Get-MajorMinorVersion -Version $sqlClientPackageVersion $sqlServerApiScanVersion = Get-MajorMinorVersion -Version $sqlServerPackageVersion @@ -381,7 +235,8 @@ Write-Host " SqlClient (family): $sqlClientApiScanVersion" Write-Host " SqlServer: $sqlServerApiScanVersion" Set-PipelineOutputVariable -Name "SqlClientPackageVersion" -Value $sqlClientPackageVersion +Set-PipelineOutputVariable -Name "SqlClientFileVersion" -Value $sqlClientFileVersion Set-PipelineOutputVariable -Name "SqlServerPackageVersion" -Value $sqlServerPackageVersion +Set-PipelineOutputVariable -Name "SqlServerFileVersion" -Value $sqlServerFileVersion Set-PipelineOutputVariable -Name "SqlClientApiScanVersion" -Value $sqlClientApiScanVersion Set-PipelineOutputVariable -Name "SqlServerApiScanVersion" -Value $sqlServerApiScanVersion -Set-PipelineOutputVariable -Name "VersionRevision" -Value $fileVersionBuildNumber diff --git a/eng/pipelines/onebranch/scripts/tests/README.md b/eng/pipelines/onebranch/scripts/tests/README.md index bda88d58a1..3585312d53 100644 --- a/eng/pipelines/onebranch/scripts/tests/README.md +++ b/eng/pipelines/onebranch/scripts/tests/README.md @@ -31,7 +31,7 @@ Invoke-Pester ./publish-symbols.Tests.ps1 -Output Detailed | Area | What's tested | | --------------------- | ---------------------------------------------------------------- | -| Version computation | Canonical output parsing, revisions, wrapping, effective package selection, and failures | +| Version computation | Canonical output parsing, effective package selection, target version composition, and failures | | Parameter validation | Empty strings rejected for all mandatory parameters | | URL construction | Base URL, register URL, request URL built from parameters | | Request bodies | Registration body, default publish flags, flag overrides | @@ -41,5 +41,6 @@ Invoke-Pester ./publish-symbols.Tests.ps1 -Output Detailed ## Notes - All external calls (`az`, `Invoke-RestMethod`) are mocked — no network access or Azure credentials are required. -- Version tests mock `dotnet`, so they do not invoke MSBuild or require a restored repository. +- Script-level version tests mock `dotnet`; package-composition tests invoke the real MSBuild + `GetVersionsSqlClient` and `GetVersionsSqlServer` targets. - Tests validate scripts in the parent directory relative to this directory. diff --git a/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 index cd38e6bb66..7a9e9779cb 100644 --- a/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 +++ b/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 @@ -4,7 +4,9 @@ #> BeforeAll { + $script:repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..' '..') $scriptPath = Join-Path $PSScriptRoot '..' 'compute-versions.ps1' + $buildProjectPath = Resolve-Path (Join-Path $script:repoRoot 'build.proj') $projectPath = Join-Path $TestDrive 'build.proj' Set-Content -LiteralPath $projectPath -Value '' @@ -19,26 +21,23 @@ BeforeAll { function Invoke-ComputeVersions { param( - [long]$Revision = 42, - [string]$BuildNumber = '', - [bool]$BuildSqlServer = $true, - [bool]$AddRevision = $true + [string]$BuildNumber = $script:testBuildNumber, + [bool]$BuildSqlServer = $true ) & $scriptPath ` -ProjectPath $projectPath ` - -Revision $Revision ` -BuildNumber $BuildNumber ` - -BuildSqlServer $BuildSqlServer ` - -AddRevision $AddRevision *>&1 | Out-String + -BuildSqlServer $BuildSqlServer *>&1 | Out-String } # Alternates between the SqlClient and SqlServer GetVersions targets, which the script always - # invokes in that order. + # invokes in that order. The versions returned here are already stamped, because Versions.props + # applies the build number before the script ever sees them. function Set-DotnetMock { param( - [string]$SqlClientPackageVersion = '7.1.0-preview3', - [string]$SqlServerPackageVersion = '1.1.0-preview1' + [string]$SqlClientPackageVersion = "7.1.0-preview3.$script:testBuildNumber", + [string]$SqlServerPackageVersion = "1.1.0-preview1.$script:testBuildNumber" ) $global:computeVersionsDotnetCallCount = 0 @@ -48,12 +47,14 @@ BeforeAll { if ($global:computeVersionsDotnetCallCount % 2 -eq 1) { return @( " PackageVersion: $SqlClientPackageVersion" + ' FileVersion: 7.1.0.26238' ' PublishedVersion: 7.0.0' ) } return @( " PackageVersion: $SqlServerPackageVersion" + ' FileVersion: 1.1.0.26238' ' PublishedVersion: 1.0.0' ) }.GetNewClosure() @@ -62,6 +63,90 @@ BeforeAll { function Set-SuccessfulDotnetMock { Set-DotnetMock } + + function Invoke-VersionTarget { + param( + [Parameter(Mandatory)] + [string]$Target, + + [Parameter(Mandatory)] + [string]$NextVersionProperty, + + [Parameter(Mandatory)] + [string]$BaseVersion, + + [string]$BuildSuffix + ) + + $arguments = @( + 'build' + $buildProjectPath + "-t:$Target" + '-v:m' + '-nologo' + "-p:BuildNumber=$script:testBuildNumber" + "-p:$NextVersionProperty=$BaseVersion" + ) + if ($BuildSuffix) { + $arguments += "-p:BuildSuffix=$BuildSuffix" + } + + $output = & dotnet @arguments 2>&1 | Out-String + if ($LASTEXITCODE -ne 0) { + throw "$Target failed with exit code ${LASTEXITCODE}:`n$output" + } + + $output + } + + # Drives PrepareForBuild rather than the validation target directly, because the hook point is + # itself the thing under test: a check wired after the compile would pass a direct invocation. + # An explicit target framework is required, as PrepareForBuild is not valid on the outer + # cross-targeting build. + function Invoke-VersionValidation { + param( + [Parameter(Mandatory)] + [string]$ProjectPath, + + [Parameter(Mandatory)] + [string]$TargetFramework, + + [string[]]$Properties = @() + ) + + $arguments = @( + 'build' + $ProjectPath + '-f' + $TargetFramework + '-t:PrepareForBuild' + '-v:m' + '-nologo' + ) + $Properties + + $output = & dotnet @arguments 2>&1 | Out-String + [pscustomobject]@{ ExitCode = $LASTEXITCODE; Output = $output } + } + + function Invoke-BuildProjTarget { + param( + [Parameter(Mandatory)] + [string]$Target, + + [string[]]$Properties = @() + ) + + $arguments = @( + 'build' + $buildProjectPath + "-t:$Target" + '-v:m' + '-nologo' + ) + $Properties + + $output = & dotnet @arguments 2>&1 | Out-String + [pscustomobject]@{ ExitCode = $LASTEXITCODE; Output = $output } + } } AfterAll { @@ -73,101 +158,218 @@ Describe 'compute-versions.ps1 Effective Versions' { Set-SuccessfulDotnetMock } - It 'appends the build number after the prerelease suffix when package revisioning is disabled' { - $output = Invoke-ComputeVersions -AddRevision $false -BuildNumber $script:testBuildNumber + It 'forwards the stamped prerelease versions from Versions.props' { + $output = Invoke-ComputeVersions $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" $output | Should -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0-preview1\.$script:testBuildNumberPattern" $output | Should -Match 'SqlClientApiScanVersion;isOutput=true]7\.1' $output | Should -Match 'SqlServerApiScanVersion;isOutput=true]1\.1' $output | Should -Match 'APIScan registration versions:\s+SqlClient \(family\): 7\.1\s+SqlServer:\s+1\.1' - $output | Should -Match "VersionRevision;isOutput=true]$script:testFileVersionBuildNumber" - } - - It 'inserts the revision before prerelease suffixes for built packages' { - $output = Invoke-ComputeVersions - - $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.42-preview3' - $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0\.42-preview1' + $output | Should -Match 'SqlClientFileVersion;isOutput=true]7\.1\.0\.26238' + $output | Should -Match 'SqlServerFileVersion;isOutput=true]1\.1\.0\.26238' } It 'retains the published SqlServer package when SqlServer is not built' { $output = Invoke-ComputeVersions -BuildSqlServer $false - $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.42-preview3' + $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0' $output | Should -Match 'SqlServerApiScanVersion;isOutput=true]1\.0' - $output | Should -Not -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0\.42' - } - - It 'wraps revisions above 65535 and logs the mapping as information' { - $output = Invoke-ComputeVersions -Revision 65536 - - $output | Should -Match 'Revision 65536.*wrapped to 1' - $output | Should -Not -Match 'task\.logissue type=warning' - $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.1-preview3' - $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0\.1-preview1' - $output | Should -Match 'VersionRevision;isOutput=true]1' - } - - It 'emits the build number rather than the wrapped revision when package revisioning is disabled' { - $output = Invoke-ComputeVersions -Revision 65536 -AddRevision $false -BuildNumber $script:testBuildNumber - - $output | Should -Not -Match 'task\.logissue type=warning' - $output | Should -Not -Match 'wrapped to' - $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" - $output | Should -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0-preview1\.$script:testBuildNumberPattern" - $output | Should -Match "VersionRevision;isOutput=true]$script:testFileVersionBuildNumber" - } + $output | Should -Not -Match "SqlServerPackageVersion;isOutput=true]1\.0\.0[\.-]$script:testFileVersionBuildNumber" - It 'retains the published SqlServer package unstamped when SqlServer is not built' { - $output = Invoke-ComputeVersions -BuildSqlServer $false -AddRevision $false -BuildNumber $script:testBuildNumber - - $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" - $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0' - $output | Should -Not -Match "SqlServerPackageVersion;isOutput=true]1\.0\.0\.$script:testFileVersionBuildNumber" + # An unbuilt SqlServer is never stamped, so it has no effective file version. + $output | Should -Match 'SqlServerFileVersion;isOutput=true](\r?\n|$)' } - It 'omits the build number from non-preview package versions when package revisioning is disabled' { + It 'forwards unstamped non-preview package versions' { Set-DotnetMock -SqlClientPackageVersion '7.1.0' -SqlServerPackageVersion '1.1.0' - $output = Invoke-ComputeVersions -AddRevision $false -BuildNumber $script:testBuildNumber + $output = Invoke-ComputeVersions $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0(\r?\n|$)' $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0(\r?\n|$)' $output | Should -Not -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0[\.-]$script:testFileVersionBuildNumber" $output | Should -Not -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0[\.-]$script:testFileVersionBuildNumber" - # The file version is still stamped so every build produces a distinct, date-encoded - # file version even for non-preview releases. - $output | Should -Match "VersionRevision;isOutput=true]$script:testFileVersionBuildNumber" + # The file version is still stamped so every build produces a date-encoded file version even + # for non-preview releases. + $output | Should -Match 'SqlClientFileVersion;isOutput=true]7\.1\.0\.26238' } +} - It 'revises non-preview package versions when package revisioning is enabled' { - Set-DotnetMock -SqlClientPackageVersion '7.1.0' -SqlServerPackageVersion '1.1.0' - - $output = Invoke-ComputeVersions - - $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.42' - $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0\.42' - $output | Should -Match 'VersionRevision;isOutput=true]42' +Describe 'GetVersions target package composition' { + It ' composes package and file versions' -ForEach @( + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0'; BuildSuffix = ''; ExpectedPackageVersion = '7.1.0' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a stable base without a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0'; BuildSuffix = 'ci'; ExpectedPackageVersion = '7.1.0-ci.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a stable base with a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0-preview3'; BuildSuffix = ''; ExpectedPackageVersion = '7.1.0-preview3.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a prerelease base without a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0-preview3'; BuildSuffix = 'ci'; ExpectedPackageVersion = '7.1.0-preview3-ci.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a prerelease base with a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0'; BuildSuffix = ''; ExpectedPackageVersion = '1.1.0' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a stable base without a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0'; BuildSuffix = 'ci'; ExpectedPackageVersion = '1.1.0-ci.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a stable base with a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0-preview1'; BuildSuffix = ''; ExpectedPackageVersion = '1.1.0-preview1.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a prerelease base without a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0-preview1'; BuildSuffix = 'ci'; ExpectedPackageVersion = '1.1.0-preview1-ci.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a prerelease base with a suffix' + } + ) { + $output = Invoke-VersionTarget ` + -Target $Target ` + -NextVersionProperty $NextVersionProperty ` + -BaseVersion $BaseVersion ` + -BuildSuffix $BuildSuffix + + $output | Should -Match "PackageVersion:\s+$([regex]::Escape($ExpectedPackageVersion))(\r?\n|$)" + $output | Should -Match "FileVersion:\s+$([regex]::Escape($ExpectedFileVersion))(\r?\n|$)" } } -Describe 'compute-versions.ps1 Error Handling' { - It 'rejects a non-positive revision' { - { Invoke-ComputeVersions -Revision 0 } | Should -Throw +Describe 'File version component validation' { + It 'rejects a four-part ' -ForEach @( + @{ + Product = 'SqlClient'; Property = 'SqlClientPackageVersion' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + Properties = @('-p:SqlClientPackageVersion=7.1.0.123') + ExpectedFileVersion = '7.1.0.123.0' + } + @{ + Product = 'SqlClient'; Property = 'SqlClientNextVersion' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + Properties = @('-p:SqlClientNextVersion=7.1.0.123', '-p:BuildNumber=1234') + ExpectedFileVersion = '7.1.0.123.1234' + } + @{ + Product = 'SqlServer'; Property = 'SqlServerPackageVersion' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + Properties = @('-p:SqlServerPackageVersion=1.1.0.123') + ExpectedFileVersion = '1.1.0.123.0' + } + @{ + Product = 'SqlServer'; Property = 'SqlServerNextVersion' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + Properties = @('-p:SqlServerNextVersion=1.1.0.123', '-p:BuildNumber=1234') + ExpectedFileVersion = '1.1.0.123.1234' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties $Properties + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '$ExpectedFileVersion' is not a four-part numeric version")) } - It 'requires a build number when package revisioning is disabled' { - Set-SuccessfulDotnetMock + It 'rejects an externally supplied file version for ' -ForEach @( + @{ + Product = 'SqlClient'; Description = 'short' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + FileVersion = '1.2' + } + @{ + Product = 'SqlClient'; Description = 'non-numeric' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + FileVersion = 'abc' + } + @{ + Product = 'SqlServer'; Description = 'short' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + FileVersion = '1.2' + } + @{ + Product = 'SqlServer'; Description = 'non-numeric' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + FileVersion = 'abc' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties @("-p:${Product}FileVersion=$FileVersion") + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '$FileVersion' is not a four-part numeric version")) + } - { Invoke-ComputeVersions -AddRevision $false } | - Should -Throw '*BuildNumber is required when AddRevision is false*' + It 'accepts the declared version' -ForEach @( + @{ + Product = 'SqlClient' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + } + @{ + Product = 'SqlServer' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties @("-p:BuildNumber=$script:testBuildNumber") + + $result.ExitCode | Should -Be 0 } +} +Describe 'build.proj file version wrappers' { + # A malformed value is used so the leaf project reports it by name, which proves the wrapper + # forwarded it verbatim without paying for a full compile. + It 'forwards through ' -ForEach @( + @{ Product = 'SqlClient'; Wrapper = 'FileVersionSqlClient'; Target = 'BuildLogging' } + @{ Product = 'SqlServer'; Wrapper = 'FileVersionSqlServer'; Target = 'BuildSqlServer' } + ) { + $result = Invoke-BuildProjTarget -Target $Target -Properties @("-p:$Wrapper=1.2") + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '1.2' is not a four-part numeric version")) + } +} + +Describe 'compute-versions.ps1 Error Handling' { It 'rejects a malformed build number' { - { Invoke-ComputeVersions -AddRevision $false -BuildNumber 'not-a-build-number' } | Should -Throw + { Invoke-ComputeVersions -BuildNumber 'not-a-build-number' } | Should -Throw + } + + It 'requires a build number' { + # Bound as empty rather than omitted; omitting a mandatory parameter prompts interactively. + { Invoke-ComputeVersions -BuildNumber '' } | Should -Throw } It 'throws when a GetVersions target fails' { @@ -188,18 +390,12 @@ Describe 'compute-versions.ps1 Error Handling' { { Invoke-ComputeVersions } | Should -Throw '*Failed to extract PackageVersion*' } - It 'throws when a revised package does not have a three-part numeric base' { - $global:computeVersionsDotnetCallCount = 0 + It 'throws when a FileVersion label is absent' { Mock -CommandName 'dotnet' -MockWith { $global:LASTEXITCODE = 0 - $global:computeVersionsDotnetCallCount++ - if ($global:computeVersionsDotnetCallCount -eq 1) { - return @('PackageVersion: 7.1-preview3') - } - - return @('PackageVersion: 1.1.0-preview1', 'PublishedVersion: 1.0.0') + return 'PackageVersion: 7.1.0-preview3.26238.3' } - { Invoke-ComputeVersions } | Should -Throw "*Expected a three-part numeric version base*" + { Invoke-ComputeVersions } | Should -Throw '*Failed to extract FileVersion*' } } diff --git a/eng/pipelines/onebranch/sqlclient-non-official.yml b/eng/pipelines/onebranch/sqlclient-non-official.yml index 3ec9bdbc75..88231ed15b 100644 --- a/eng/pipelines/onebranch/sqlclient-non-official.yml +++ b/eng/pipelines/onebranch/sqlclient-non-official.yml @@ -28,13 +28,6 @@ parameters: type: boolean default: false - # Append the Build.BuildId revision to package and file versions to reduce collisions when - # repeatedly publishing tests to symbol servers or the NuGet test feed. - - name: addRevision - displayName: Use the build ID as the revision version - type: boolean - default: false - # True to publish symbols to private and public servers. - name: publishSymbols displayName: Publish symbols @@ -256,8 +249,12 @@ extends: sbom: enabled: true - packageName: 'Microsoft.Data.SqlClient' - packageVersion: '$(Build.BuildNumber)' + # OneBranch resolves these from globalSdl only -- there is no per-job form -- so they + # indirect through variables that each build job sets to the package it produces. Jobs + # that publish no packages disable SBOM via ob_sdl_sbom_enabled rather than defaulting + # these. See build-buildproj-job.yml. + packageName: '$(sbomPackageName)' + packageVersion: '$(sbomPackageVersion)' tsa: # TSA (Trust Services Automation) files SDL analysis findings as Azure DevOps bug work @@ -297,7 +294,6 @@ extends: - template: /eng/pipelines/onebranch/stages/compute-versions-stage.yml@self parameters: - addRevision: ${{ parameters.addRevision }} buildSqlServer: ${{ parameters.buildSqlServer }} - template: /eng/pipelines/onebranch/stages/build-stages.yml@self diff --git a/eng/pipelines/onebranch/sqlclient-official.yml b/eng/pipelines/onebranch/sqlclient-official.yml index 2bc98a655b..40b08c76c0 100644 --- a/eng/pipelines/onebranch/sqlclient-official.yml +++ b/eng/pipelines/onebranch/sqlclient-official.yml @@ -37,12 +37,6 @@ parameters: type: boolean default: false - # Append the Build.BuildId revision to package and file versions. - - name: addRevision - displayName: Use the build ID as the revision version - type: boolean - default: false - # True to publish symbols to private and public servers. - name: publishSymbols displayName: Publish symbols @@ -270,8 +264,12 @@ extends: sbom: enabled: true - packageName: 'Microsoft.Data.SqlClient' - packageVersion: '$(Build.BuildNumber)' + # OneBranch resolves these from globalSdl only -- there is no per-job form -- so they + # indirect through variables that each build job sets to the package it produces. Jobs + # that publish no packages disable SBOM via ob_sdl_sbom_enabled rather than defaulting + # these. See build-buildproj-job.yml. + packageName: '$(sbomPackageName)' + packageVersion: '$(sbomPackageVersion)' tsa: # TSA (Trust Services Automation) files SDL analysis findings as Azure DevOps bug work @@ -306,7 +304,6 @@ extends: - template: /eng/pipelines/onebranch/stages/compute-versions-stage.yml@self parameters: - addRevision: ${{ parameters.addRevision }} buildSqlServer: ${{ parameters.buildSqlServer }} - template: /eng/pipelines/onebranch/stages/build-stages.yml@self diff --git a/eng/pipelines/onebranch/stages/build-stages.yml b/eng/pipelines/onebranch/stages/build-stages.yml index 98e656004b..afd1907859 100644 --- a/eng/pipelines/onebranch/stages/build-stages.yml +++ b/eng/pipelines/onebranch/stages/build-stages.yml @@ -80,8 +80,10 @@ stages: dependsOn: compute_versions variables: - - name: versionRevision - value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlServerFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerFileVersion'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion @@ -107,7 +109,7 @@ stages: signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' dependencies: [] - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient.Internal.Logging' packageShortName: 'Logging' versionPropertySuffix: 'SqlClient' @@ -130,7 +132,7 @@ stages: signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' dependencies: [] - revision: '$(versionRevision)' + fileVersion: '$(sqlServerFileVersion)' packageFullName: 'Microsoft.SqlServer.Server' packageShortName: 'SqlServer' versionPropertySuffix: 'SqlServer' @@ -148,8 +150,8 @@ stages: - build_independent variables: - - name: versionRevision - value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlClientApiScanVersion @@ -174,7 +176,7 @@ stages: - artifactName: '${{ parameters.loggingArtifactsName }}' shortName: 'Logging' version: '$(sqlClientPackageVersion)' - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient.Extensions.Abstractions' packageShortName: 'Abstractions' versionPropertySuffix: 'SqlClient' @@ -192,8 +194,8 @@ stages: - build_abstractions variables: - - name: versionRevision - value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion @@ -234,7 +236,7 @@ stages: - artifactName: '' shortName: 'SqlServer' version: '$(sqlServerPackageVersion)' - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient' packageShortName: 'SqlClient' versionPropertySuffix: 'SqlClient' @@ -261,7 +263,7 @@ stages: - artifactName: '${{ parameters.loggingArtifactsName }}' shortName: 'Logging' version: '$(sqlClientPackageVersion)' - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient.Extensions.Azure' packageShortName: 'Azure' versionPropertySuffix: 'SqlClient' @@ -278,8 +280,8 @@ stages: - build_dependent variables: - - name: versionRevision - value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion @@ -322,7 +324,7 @@ stages: - artifactName: '' shortName: 'SqlServer' version: '$(sqlServerPackageVersion)' - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' packageShortName: 'AkvProvider' versionPropertySuffix: 'SqlClient' diff --git a/eng/pipelines/onebranch/stages/compute-versions-stage.yml b/eng/pipelines/onebranch/stages/compute-versions-stage.yml index 11e2187e57..9514c0d56e 100644 --- a/eng/pipelines/onebranch/stages/compute-versions-stage.yml +++ b/eng/pipelines/onebranch/stages/compute-versions-stage.yml @@ -17,20 +17,14 @@ # - Microsoft.SqlServer.Server is versioned separately: it uses its "next" version when it is # being built this run, or its "published" (last shipped to NuGet) version when it is not built # (so the SqlClient family depends on the most recently published SqlServer package). -# - Runs use their "next" version from Versions.props. By default the pipeline build number is -# appended to prerelease package versions (7.1.0-preview3.26238.3) and its date segment becomes -# the file-version build number (7.1.0.26238), matching the shape shipped by earlier previews. -# Runs can instead append the Build.BuildId revision, reducing collisions when repeatedly -# publishing the same base version for testing. +# - Runs use their "next" version from Versions.props. The pipeline build number is appended to +# prerelease package versions (7.1.0-preview3.26238.3) and its date segment becomes the +# file-version build number (7.1.0.26238), matching the shape shipped by earlier previews. # - Versions are extracted via build.proj GetVersions* targets by parsing stdout. # # This stage MUST run before all build stages so they can consume computed versions. parameters: - # Whether to append the Build.BuildId revision to package and file versions. - - name: addRevision - type: boolean - # Whether Microsoft.SqlServer.Server is being built this run. This drives the SqlServer version # selection (next when built, published when not); the SqlClient family always uses its next # version. When SqlServer is not built, the SqlClient family depends on the published SqlServer @@ -54,6 +48,8 @@ stages: ob_sdl_apiscan_enabled: false ob_sdl_binskim_break: false ob_sdl_binskim_enabled: false + # No packages are published here, so there is nothing to describe in an SBOM. + ob_sdl_sbom_enabled: false steps: - pwsh: | @@ -69,8 +65,8 @@ stages: # may run during the GetVersions* evaluation. - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - # Extract canonical versions, resolve the effective built/published values, optionally - # add the run revision, and publish the downstream stage output variables. + # Extract canonical versions, resolve the effective built/published values, and publish + # the downstream stage output variables. - task: PowerShell@2 displayName: "Compute Effective Versions" name: "versions" @@ -80,7 +76,5 @@ stages: filePath: $(Build.SourcesDirectory)/eng/pipelines/onebranch/scripts/compute-versions.ps1 arguments: >- -ProjectPath "$(Build.SourcesDirectory)/build.proj" - -Revision "$(Build.BuildId)" -BuildNumber "$(Build.BuildNumber)" -BuildSqlServer $${{ parameters.buildSqlServer }} - -AddRevision $${{ parameters.addRevision }} diff --git a/eng/pipelines/onebranch/steps/build-buildproj-step.yml b/eng/pipelines/onebranch/steps/build-buildproj-step.yml index 5cfc809772..961b972f8c 100644 --- a/eng/pipelines/onebranch/steps/build-buildproj-step.yml +++ b/eng/pipelines/onebranch/steps/build-buildproj-step.yml @@ -37,8 +37,8 @@ parameters: - SqlClient - SqlServer - # Version revision translated to build.proj's BuildNumber property at this boundary. - - name: revision + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion type: string # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this @@ -75,7 +75,7 @@ steps: -p:ReferenceType=Package -p:SkipDependencyPack=true -p:SigningKeyPath="$(keyFile.secureFilePath)" - -p:BuildNumber="${{ parameters.revision }}" + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" ${{ parameters.dependencyArguments }} diff --git a/eng/pipelines/onebranch/steps/pack-buildproj-step.yml b/eng/pipelines/onebranch/steps/pack-buildproj-step.yml index 4960872d7b..b8ee6e72dd 100644 --- a/eng/pipelines/onebranch/steps/pack-buildproj-step.yml +++ b/eng/pipelines/onebranch/steps/pack-buildproj-step.yml @@ -49,8 +49,8 @@ parameters: - SqlClient - SqlServer - # Version revision translated to build.proj's BuildNumber property at this boundary. - - name: revision + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion type: string # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this @@ -79,7 +79,7 @@ steps: -p:Configuration=${{ parameters.buildConfiguration }} -p:PackBuild=false -p:ReferenceType=Package - -p:BuildNumber="${{ parameters.revision }}" + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" ${{ parameters.dependencyArguments }} diff --git a/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml b/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml index 9f58773174..6e9b0e8106 100644 --- a/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml +++ b/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml @@ -129,11 +129,8 @@ parameters: - SqlClient - SqlServer - # The three parameters below mirror build-buildproj-step.yml so the analysis build resolves the - # same package versions as the real build. - # - # Version revision translated to build.proj's BuildNumber property at this boundary. - - name: revision + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion type: string # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this @@ -165,7 +162,7 @@ steps: -p:Configuration=Release -p:ReferenceType=Package -p:SkipDependencyPack=true - -p:BuildNumber="${{ parameters.revision }}" + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" -p:IsolatedBuildPath="$(Agent.TempDirectory)/roslyn" -p:EnableAnalyzers=true diff --git a/eng/pipelines/onebranch/variables/package-variables.yml b/eng/pipelines/onebranch/variables/package-variables.yml index 62afca6605..65fd0867d0 100644 --- a/eng/pipelines/onebranch/variables/package-variables.yml +++ b/eng/pipelines/onebranch/variables/package-variables.yml @@ -7,23 +7,6 @@ # This file contains variables that relate to the various packages that are produced via the # OneBranch official/non-official pipelines. They are grouped by the packages they represent. # -# VERSION STRATEGY -# ================ -# Package versions are computed by Versions.props using one input: -# - BuildNumber ($(Build.BuildNumber), provided by ADO) -# -# The SqlClient *family* (Internal.Logging, Extensions.Abstractions, Microsoft.Data.SqlClient, -# Extensions.Azure, and the AlwaysEncrypted.AzureKeyVaultProvider) shares a single version defined -# by SqlClientNextVersion in src/Microsoft.Data.SqlClient/Versions.props. Microsoft.SqlServer.Server -# is versioned separately by SqlServerNextVersion. -# -# The *NextVersion properties define the base version: -# - On main: X.Y.Z-preview1 → official produces "X.Y.Z-preview1" -# - On release/X.Y: X.Y.Z → official produces "X.Y.Z" -# -# No per-package version variables are needed here — Versions.props is the single source. -# Downstream stages obtain the computed versions via the compute-versions stage. -# # ARTIFACT NAMING # =============== # OneBranch automatically publishes pipeline artifacts from all jobs named as: diff --git a/src/Microsoft.Data.SqlClient/Versions.props b/src/Microsoft.Data.SqlClient/Versions.props index f67997a10f..38df551f3b 100644 --- a/src/Microsoft.Data.SqlClient/Versions.props +++ b/src/Microsoft.Data.SqlClient/Versions.props @@ -31,7 +31,6 @@ 7.1.0-preview3 @@ -50,13 +49,8 @@ not valid in a file version. Start with the suffix-free package base and append the configured file-version build number, which converts a three-part package version such as 1.2.3-preview1 to 1.2.3.42. - - A revision-enabled pipeline supplies a four-part package base such as 1.2.3.42. That base - is already a complete file version, so the following conditional assignment overrides the - fallback instead of producing an invalid five-part version. --> $(SqlClientPackageVersionBase).$(FileVersionBuildNumber) - $(SqlClientPackageVersionBase) @@ -65,18 +59,24 @@ When a build number is provided, the default version is used as the base of the package and file versions. - If a build suffix is provided, this is appended to the package version. This is meant to be - used by automated pre-release systems to indicate the source of the build. + If a build suffix is provided, it is appended as a prerelease tag. This is meant to be used + by automated pre-release systems to indicate the source of the build. - If a build suffix is not provided, no pre-release tag will be added to package version. If - the default version already has a pre-release tag added to it (eg, "7.0.0-preview1") this - will be retained for the package version, but will be stripped off for the file version - (letters are not allowed in file/assembly versions). This is meant to be used by official - build pipelines to generate production-ready builds. + The build number is then appended to any version carrying a prerelease tag — whether that + tag came from the declared version or from the build suffix — so every automated run + produces a distinct package version (eg, "7.0.0-preview1" becomes + "7.0.0-preview1.26238.3"). A version with no prerelease tag is left exactly as declared, + because released packages are not stamped with a build number. The tag is always stripped + for the file version (letters are not allowed in file/assembly versions). This is meant to + be used by official build pipelines to generate production-ready builds. --> - $(SqlClientNextVersion)-$(BuildSuffix)$(BuildNumber) - $(SqlClientNextVersion) + <_SqlClientCandidateVersion>$(SqlClientNextVersion) + <_SqlClientCandidateVersion Condition="'$(BuildSuffix)' != ''">$(SqlClientNextVersion)-$(BuildSuffix) + + + $(_SqlClientCandidateVersion).$(BuildNumber) + $(_SqlClientCandidateVersion) $(SqlClientNextVersion.Split('-')[0]).$(FileVersionBuildNumber) @@ -105,4 +105,11 @@ $(SqlClientFileVersion.Split('.')[0]).0.0.0 + + + + + diff --git a/src/Microsoft.SqlServer.Server/Versions.props b/src/Microsoft.SqlServer.Server/Versions.props index e03f8d2967..a83b5d246a 100644 --- a/src/Microsoft.SqlServer.Server/Versions.props +++ b/src/Microsoft.SqlServer.Server/Versions.props @@ -45,13 +45,8 @@ not valid in a file version. Start with the suffix-free package base and append the configured file-version build number, which converts a three-part package version such as 1.2.3-preview1 to 1.2.3.42. - - A revision-enabled pipeline supplies a four-part package base such as 1.2.3.42. That base - is already a complete file version, so the following conditional assignment overrides the - fallback instead of producing an invalid five-part version. --> $(SqlServerPackageVersionBase).$(FileVersionBuildNumber) - $(SqlServerPackageVersionBase) @@ -60,18 +55,24 @@ When a build number is provided, the default version is used as the base of the package and file versions. - If a build suffix is provided, this is appended to the package version. This is meant to be - used by automated pre-release systems to indicate the source of the build. + If a build suffix is provided, it is appended as a prerelease tag. This is meant to be used + by automated pre-release systems to indicate the source of the build. - If a build suffix is not provided, no pre-release tag will be added to package version. If - the default version already has a pre-release tag added to it (eg, "7.0.0-preview1") this - will be retained for the package version, but will be stripped off for the file version - (letters are not allowed in file/assembly versions). This is meant to be used by official - build pipelines to generate production-ready builds. + The build number is then appended to any version carrying a prerelease tag — whether that + tag came from the declared version or from the build suffix — so every automated run + produces a distinct package version (eg, "1.1.0-preview1" becomes + "1.1.0-preview1.26238.3"). A version with no prerelease tag is left exactly as declared, + because released packages are not stamped with a build number. The tag is always stripped + for the file version (letters are not allowed in file/assembly versions). This is meant to + be used by official build pipelines to generate production-ready builds. --> - $(SqlServerNextVersion)-$(BuildSuffix)$(BuildNumber) - $(SqlServerNextVersion) + <_SqlServerCandidateVersion>$(SqlServerNextVersion) + <_SqlServerCandidateVersion Condition="'$(BuildSuffix)' != ''">$(SqlServerNextVersion)-$(BuildSuffix) + + + $(_SqlServerCandidateVersion).$(BuildNumber) + $(_SqlServerCandidateVersion) $(SqlServerNextVersion.Split('-')[0]).$(FileVersionBuildNumber) @@ -100,4 +101,11 @@ $(SqlServerFileVersion.Split('.')[0]).0.0.0 + + + + + From 76ce4329434ec0ea6f648267c70732970d688f09 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:25:28 -0700 Subject: [PATCH 27/51] [Scheduled Run] Localized resource files from OneLocBuild (#4651) Co-authored-by: SqlClient DevOps --- src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx | 2 +- src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx index d333b05fd5..bd22fd385c 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx @@ -2083,7 +2083,7 @@ Pokud je klíčové slovo Integrated Security připojovacího řetězce nastavené na hodnotu true nebo SSPI, nejde nastavit vlastnost AccessTokenCallback. - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + Nelze nastavit více než jednu z vlastností AccessToken, AccessTokenCallback nebo SspiContextProvider. Pokud je v připojovacím řetězci nastavená možnost Authentication=Active Directory Default, nejde nastavit vlastnost AccessTokenCallback. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx index 36003faa0f..5570cfa5f4 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx @@ -2083,7 +2083,7 @@ AccessTokenCallback-Eigenschaft kann nicht festgelegt werden, wenn das Schlüsselwort für Verbindungszeichenfolgen 'Integrated Security' auf 'true' oder 'SSPI' gesetzt wurde. - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + Es kann nur eine der Eigenschaften AccessToken, AccessTokenCallback oder SspiContextProvider festgelegt werden. Die Eigenschaft AccessTokenCallback kann nicht festgelegt werden, wenn in der Verbindungszeichenfolge "Authentication=Active Directory Default" angegeben wurde. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx index f24a37eff9..1f84064549 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx @@ -2083,7 +2083,7 @@ No se puede establecer la propiedad AccessTokenCallback si la palabra clave de cadena de conexión "Integrated Security" se ha establecido en "true" o "SSPI". - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + No se puede establecer más de una de las propiedades AccessToken, AccessTokenCallback o SspiContextProvider. No se puede establecer la propiedad AccessTokenCallback si se ha especificado 'Authentication=Active Directory Default' en la cadena de conexión. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx index 32a65bfc0d..537c0dd0cf 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx @@ -2083,7 +2083,7 @@ Impossible de définir la propriété AccessTokenCallback si le mot clé de chaîne de connexion « Sécurité intégrée » a été défini sur « true » ou « SSPI ». - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + Nous ne pouvons pas définir plus d’une des propriétés AccessToken, AccessTokenCallback ou SspiContextProvider. Impossible de définir la propriété AccessTokenCallback si 'Authentication=Active Directory Default' a été spécifié dans la chaîne de connexion. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx index 4109e2f808..8310513c43 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx @@ -2083,7 +2083,7 @@ Non è possibile impostare la proprietà AccessTokenCallback se la parola chiave della stringa di connessione 'Integrated Security' è stata impostata su 'true' o 'SSPI'. - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + Non è possibile impostare più di una delle proprietà AccessToken, AccessTokenCallback o SspiContextProvider. Impossibile impostare la proprietà AccessTokenCallback se nella stringa di connessione è stato specificato 'Authentication=Active Directory Default'. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx index b0ff96812b..f24fe2d297 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx @@ -2083,7 +2083,7 @@ 'Integrated Security' 接続文字列キーワードが 'true' または 'SSPI' に設定されている場合、AccessTokenCallback プロパティを設定できません。 - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + AccessToken、AccessTokenCallback、 SspiContextProvider のプロパティを 2 つ以上設定することはできません。 接続文字列に 'Authentication=Active Directory Default' が指定されている場合、AccessTokenCallback プロパティを設定できません。 diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx index 62433d0ea0..ab87cea583 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx @@ -2083,7 +2083,7 @@ 'Integrated Security' 연결 문자열 키워드가 'true' 또는 'SSPI'로 설정된 경우 AccessTokenCallback 속성을 설정할 수 없습니다. - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + AccessToken, AccessTokenCallback 또는 SspiContextProvider 속성 중 두 개 이상을 설정할 수 없습니다. 연결 문자열에 'Authentication=Active Directory Default'가 지정된 경우 AccessTokenCallback 속성을 설정할 수 없습니다. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx index c410e654a0..dcc088af86 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx @@ -2083,7 +2083,7 @@ Nie można ustawić właściwości AccessTokenCallback, jeśli słowo kluczowe parametrów połączenia „Integrated Security” ma wartość „true” lub „SSPI”. - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + Nie można ustawić więcej niż jednej właściwości AccessToken, AccessTokenCallback lub SspiContextProvider. Nie można ustawić właściwości AccessTokenCallback, jeśli w parametrach połączenia określono wartość „Authentication=Active Directory Default”. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx index 31d3ffc029..458096ee2a 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx @@ -2083,7 +2083,7 @@ Não é possível definir a propriedade AccessTokenCallback se a palavra-chave da cadeia de conexão 'Integrated Security' tiver sido definida como 'true' ou 'SSPI'. - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + Não é possível definir mais de uma das propriedades AccessToken, AccessTokenCallback ou SspiContextProvider. Não é possível definir a propriedade AccessTokenCallback se 'Authentication=Active Directory Default' tiver sido especificado na cadeia de conexão. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx index d5d4039980..4a14f91101 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx @@ -2083,7 +2083,7 @@ Если ключевому слову строки подключения "Integrated Security" задано значение "true" или "SSPI", свойство "AccessTokenCallback" задать не удастся. - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + Не удается установить несколько свойств AccessToken, AccessTokenCallback или SspiContextProvider. Если в строке подключения указан параметр "Authentication=Active Directory Default", свойство AccessTokenCallback задать не удастся. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx index c62b087f11..60698762e5 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx @@ -2083,7 +2083,7 @@ 'Integrated Security' bağlantı dizesi anahtar sözcüğü 'true' veya 'SSPI' olarak ayarlanmışsa AccessTokenCallback özelliği ayarlanamaz. - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + AccessToken, AccessTokenCallback veya SspiContextProvider özelliklerinden birden fazlası ayarlanamaz. Bağlantı dizesinde 'Authentication=Active Directory Default' belirtilmişse AccessTokenCallback özelliği ayarlanamaz. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx index 69ab4523bc..2c6388df6e 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx @@ -2083,7 +2083,7 @@ 如果 "Integrated Security" 连接字符串关键字设置为 "true" 或 "SSPI",则无法设置 AccessTokenCallback 属性。 - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + 只能设置以下属性其中之一: AccessToken、AccessTokenCallback 或 SspiContextProvider。 如果在连接字符串中指定了 "Authentication=Active Directory Default",则无法设置 AccessTokenCallback 属性。 diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx index a5fc02dd7b..d71c496a85 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx @@ -2083,7 +2083,7 @@ 如果 'Integrated Security' 連接字串關鍵字已設定為 'true' 或 'SSPI',就不能設定 AccessTokenCallback 屬性。 - Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. + AccessToken、AccessTokenCallback 或 SspiContextProvider 這些屬性中,不能設定超過一個。 如果已在連接字串中指定 'Authentication=Active Directory Default',就不能設定 AccessTokenCallback 屬性。 From 88a591a401994bbbd9f20356778e0b6209561d5a Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:30:01 -0300 Subject: [PATCH 28/51] Categorize Entra integrated test as interactive (#4510) --- .../ado-pipelines.instructions.md | 2 +- .github/instructions/testing.instructions.md | 16 +++++++-------- .../triage-pipeline-failures.prompt.md | 4 ++-- TESTGUIDE.md | 5 +++++ build.proj | 20 +++++++++++++++++++ .../jobs/test-azure-package-ci-job.yml | 7 ------- .../stages/build-azure-package-ci-stage.yml | 3 --- .../Azure/test/AADConnectionTest.cs | 17 +++------------- .../test/ActiveDirectoryInteractiveTests.cs | 2 +- .../Azure/test/Config.cs | 8 ++++---- .../tests/FunctionalTests/LocalizationTest.cs | 2 +- .../FunctionalTests/SqlDataRecordTest.cs | 2 +- .../ManualTests/AlwaysEncrypted/ApiShould.cs | 4 ++-- ...llFromReaderConnectionCloseOnEventAsync.cs | 2 +- .../SQL/AdapterTest/AdapterTest.cs | 2 +- .../SQL/AsyncTest/AsyncTimeoutTest.cs | 2 +- .../ConnectionPoolTest/ConnectionPoolTest.cs | 2 +- .../ConnectionPoolTest/TransactionPoolTest.cs | 2 +- .../SQL/DataReaderTest/DataReaderTest.cs | 2 +- .../SQL/DataStreamTest/DataStreamTest.cs | 2 +- .../MarsSessionPoolingTest.cs | 6 +++--- .../SQL/ParameterTest/DateTimeVariantTests.cs | 2 +- .../SQL/ParameterTest/ParametersTest.cs | 10 +++++----- .../ParameterTest/SqlVariantParameterTests.cs | 6 +++--- .../StreamInputParameterTests.cs | 2 +- .../ParameterTest/TvpColumnBoundariesTests.cs | 2 +- .../SqlNotificationTest.cs | 2 +- .../DistributedTransactionTest.cs | 2 +- .../SQL/UdtTest/SqlServerTypesTest.cs | 8 ++++---- .../WeakRefTestYukonSpecific.cs | 2 +- .../TracingTests/DiagnosticTest.cs | 2 +- .../ManualTests/TracingTests/MetricsTest.cs | 4 ++-- .../ChannelDbConnectionPoolTest.cs | 2 +- .../WaitHandleDbConnectionPoolShutdownTest.cs | 2 +- ...itHandleDbConnectionPoolTransactionTest.cs | 2 +- .../ConnectionFailoverTests.cs | 6 +++--- .../SimulatedServerTests/ConnectionTests.cs | 4 ++-- .../config.default.jsonc | 6 ++++++ 38 files changed, 93 insertions(+), 83 deletions(-) diff --git a/.github/instructions/ado-pipelines.instructions.md b/.github/instructions/ado-pipelines.instructions.md index 3e5a66af78..4fd07cfc88 100644 --- a/.github/instructions/ado-pipelines.instructions.md +++ b/.github/instructions/ado-pipelines.instructions.md @@ -93,7 +93,7 @@ Test filters — default excludes `failing` and `flaky` categories: - `nonwindowstests` / `nonlinuxtests` — OS-specific exclusions Flaky test quarantine: -- Quarantined tests (`[Trait("Category", "flaky")]`) run in separate steps after main tests +- Quarantined tests (`[Trait("category", "flaky")]`) run in separate steps after main tests - Main test runs are not blocked by flaky failures - No code coverage collected for flaky runs - Configured in `common/templates/steps/build-and-run-tests-netcore-step.yml`, `build-and-run-tests-netfx-step.yml`, and `run-all-tests-step.yml` diff --git a/.github/instructions/testing.instructions.md b/.github/instructions/testing.instructions.md index df6b909f8a..9f4032e070 100644 --- a/.github/instructions/testing.instructions.md +++ b/.github/instructions/testing.instructions.md @@ -69,7 +69,7 @@ Copy `config.default.jsonc` to `config.jsonc` and configure: ## Test Categories and Attributes ### Category Exclusions -Use `[Trait("Category", "...")]` (xUnit, used in both ManualTests and UnitTests) to mark test categories and exclusions: +Use `[Trait("category", "...")]` (xUnit, used in both ManualTests and UnitTests) to mark test categories and exclusions: | Category | Excluded On | Description | |----------|-------------|-------------| @@ -82,7 +82,7 @@ Use `[Trait("Category", "...")]` (xUnit, used in both ManualTests and UnitTests) | `flaky` | All platforms (quarantine) | Intermittently failing tests (see Quarantine Zone below) | ### Flaky Test Quarantine Zone -Tests that intermittently fail are quarantined with `[Trait("Category", "flaky")]`. Quarantined tests: +Tests that intermittently fail are quarantined with `[Trait("category", "flaky")]`. Quarantined tests: - Are **excluded** from regular test runs by the default filter: `category!=failing&category!=flaky` - Run in **separate quarantine pipeline steps** to track their status without blocking CI - Do **not** collect code coverage @@ -96,16 +96,16 @@ Tests that intermittently fail are quarantined with `[Trait("Category", "flaky") **How to quarantine:** ```csharp // For unit tests (xUnit Trait) -[Trait("Category", "flaky")] +[Trait("category", "flaky")] public class FlakyConnectionTests { ... } // For individual test methods -[Trait("Category", "flaky")] +[Trait("category", "flaky")] [ConditionalFact(...)] public async Task OpenAsync_TimingDependent_MayFail() { ... } ``` -**How to un-quarantine:** Remove the `[Trait("Category", "flaky")]` attribute once the root cause is fixed and the test passes consistently. +**How to un-quarantine:** Remove the `[Trait("category", "flaky")]` attribute once the root cause is fixed and the test passes consistently. ### Test Timeout Enforcement All test runs use `--blame-hang-timeout 10m` to kill tests that hang for more than 10 minutes. This is configured in `build.proj` and applied to all test targets. If a test is expected to run longer than 10 minutes, it must be restructured or split. @@ -120,11 +120,11 @@ This can be overridden via build property: `dotnet build build.proj -t:TestSqlCl ### Test Attributes ```csharp // Platform-specific exclusion -[Trait("Category", "nonlinuxtests")] +[Trait("category", "nonlinuxtests")] public void TestWindowsSpecificFeature() { ... } // Skip on .NET Framework -[Trait("Category", "nonnetfxtests")] +[Trait("category", "nonnetfxtests")] public void TestNetCoreOnlyFeature() { ... } // Conditional skip based on test configuration @@ -132,7 +132,7 @@ public void TestNetCoreOnlyFeature() { ... } public void TestRequiresDatabase() { ... } // Quarantined flaky test -[Trait("Category", "flaky")] +[Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] public void TestIntermittentlyFails() { ... } ``` diff --git a/.github/prompts/triage-pipeline-failures.prompt.md b/.github/prompts/triage-pipeline-failures.prompt.md index c4ff676d17..98c223c3d5 100644 --- a/.github/prompts/triage-pipeline-failures.prompt.md +++ b/.github/prompts/triage-pipeline-failures.prompt.md @@ -150,7 +150,7 @@ immediately-preceding in-scope build (the parent commit). A failure present befo ## Step 7 — Check quarantine status before acting -A test is already quarantined if it carries `[Trait("Category", "flaky")]`; those run +A test is already quarantined if it carries `[Trait("category", "flaky")]`; those run in a separate, non-blocking quarantine step (`TestFilters="category=flaky"`) while the regular step excludes `category!=failing&category!=flaky&category!=interactive`. @@ -175,7 +175,7 @@ the user which items to act on. For each item the user approves: 1. Prefer a **deterministic fix** that removes the race/isolation/timing dependency. -2. Otherwise **quarantine**: add `[Trait("Category", "flaky")]` plus a comment holding +2. Otherwise **quarantine**: add `[Trait("category", "flaky")]` plus a comment holding the observed failure signature (test name, assertion, key stack frames) and the root-cause reasoning. Mirror the style of existing quarantine comments in the test suite. 3. Cover both sync and async variants when the API has both. diff --git a/TESTGUIDE.md b/TESTGUIDE.md index c813db0345..206092b597 100644 --- a/TESTGUIDE.md +++ b/TESTGUIDE.md @@ -220,6 +220,7 @@ Update `config.jsonc` for your environment before running manual tests. The most "NPConnectionString": "Data Source=np:localhost;Database=Northwind;Integrated Security=true;Encrypt=false;", "EnclaveEnabled": false, "TracingEnabled": false, + "SupportsEntraIntegrated": false, "SupportsIntegratedSecurity": true } ``` @@ -247,6 +248,9 @@ dotnet build -t:TestSqlClientManual -p:TestSet=2 ## Configuration Properties +`SupportsEntraIntegrated` applies to Azure SQL Database, Azure SQL Managed Instance, and SQL Server +2022 or later configured for Microsoft Entra authentication through Azure Arc. + | Property | Description | Example or notes | |----------------------------------|---------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| | `TCPConnectionString` | Connection string for a TCP-enabled SQL Server or Azure SQL database. | `Data Source=tcp:localhost;Database=Northwind;Integrated Security=true;Encrypt=false;` | @@ -262,6 +266,7 @@ dotnet build -t:TestSqlClientManual -p:TestSet=2 | `AADServicePrincipalSecret` | Optional application secret for service-principal authentication tests. | Keep this only in local, ignored config files or secure pipeline variables. | | `AzureKeyVaultURL` | Optional Azure Key Vault URL for Always Encrypted tests. | `https://.vault.azure.net/` | | `AzureKeyVaultTenantId` | Optional Entra ID tenant ID for Azure Key Vault tests. | Tenant ID GUID. | +| `SupportsEntraIntegrated` | Whether the target supports Entra Integrated authentication for the Windows identity. | `true` or `false`; defaults to `false`. See supported targets above. | | `SupportsIntegratedSecurity` | Whether the user running tests has integrated-security access to the target SQL Server. | `true` or `false`. | | `LocalDbAppName` | Optional LocalDB instance name. Empty disables LocalDB testing. | `MSSQLLocalDB` or another local instance. | | `LocalDbSharedInstanceName` | Optional shared LocalDB instance name. | Used only when testing shared LocalDB. | diff --git a/build.proj b/build.proj index 2e19233b2a..4dfef2b77b 100644 --- a/build.proj +++ b/build.proj @@ -401,6 +401,26 @@ + using System; + using System.Collections.Generic; + using System.Data; using Microsoft.Data.SqlClient.Server; - - [Microsoft.Data.SqlClient.Server.SqlProcedure] - public static void CreateNewRecord() + + // Stream rows to SQL Server as a table-valued parameter. + public static IEnumerable<SqlDataRecord> CreateNewRecord() { - - // Variables. + // Re-use a single SqlDataRecord instance rather than allocating a new one for each row. + // Each row's values are read before SqlCommand advances to the next one. SqlDataRecord record; // Create a new record with the column metadata. The constructor is @@ -38,8 +41,12 @@ record.SetInt32(1, 42); record.SetDateTime(2, DateTime.Now); - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Stream the first record to SQL Server. + yield return record; + + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } @@ -75,9 +82,6 @@ // Set the record fields. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); - - // Send the record to the calling program. - SqlContext.Pipe.Send(record); diff --git a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml index cbabfd8cb0..492e85e46b 100644 --- a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml +++ b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml @@ -1,19 +1,24 @@ - + - Specifies and retrieves metadata information from parameters and columns of objects. This class cannot be inherited. + Specifies the metadata information of a column used by a object. This class cannot be inherited. + + instances are typically built once and reused to construct many objects which are streamed to SQL Server as a table-valued parameter. For more information, see Table-Valued Parameters. + - The following example shows the creation of several objects, which describe the column metadata of a record, and the creation of a . The column values of the are set and the is sent to the calling program using the class. + The following example shows the creation of several objects, which describe the column metadata of a record, and the generation of a stream of records. These records can be streamed to SQL Server as a table-valued parameter by assigning the return value of the method to the property. + using System; + using System.Collections.Generic; + using System.Data; using Microsoft.Data.SqlClient.Server; - - [Microsoft.Data.SqlClient.Server.SqlProcedure] - public static void CreateNewRecord() + + public static IEnumerable<SqlDataRecord> ReturnNewRecords() { // Variables. SqlMetaData column1Info; @@ -32,13 +37,15 @@ column2Info, column3Info }); - // Set the record fields. + // Set the fields of the first record and stream it to SQL Server. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); record.SetDateTime(2, DateTime.Now); + yield return record; - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } @@ -456,7 +463,7 @@ The SQL Server type name for . - Initializes a new instance of the class with the specified column name, user-defined type (UDT), and SQLServer type. + Initializes a new instance of the class with the specified column name, user-defined type (UDT), and SQL Server type. From 5b34b39fa1480b5327e4582ff3222e510247b5ee Mon Sep 17 00:00:00 2001 From: priyankatiwari08 Date: Thu, 10 Sep 2026 14:51:13 +0530 Subject: [PATCH 31/51] Fix | Preserve delegated transactions when resetting a pooled connection (#4557) * Fix | Preserve delegated transactions when resetting a pooled connection Fixes #4001 A connection can be tied to a transaction in one of two mutually exclusive ways on this code path: - It is the *root* of a delegated transaction. The transaction has been delegated down to this connection, so IsTransactionRoot is true and EnlistedTransaction is null. - It merely *enlisted* in a transaction owned elsewhere, so EnlistedTransaction is set and IsTransactionRoot is false. Before #3019, ResetConnection() only preserved the transaction for the delegated-root case, which missed the enlisted case (#2970). PR #3019 replaced that check with `EnlistedTransaction is not null` rather than adding to it, which fixed #2970 but silently dropped the delegated-root case. The result is that a connection returned to the pool while it is still the root of a live delegated transaction has its server-side transaction reset out from under System.Transactions. When the TransactionScope later rolls back, SqlDelegatedTransaction.Rollback fails and dooms the connection. With a small pool the same doomed physical connection is handed straight back out, producing "The requested operation cannot be completed because the connection has been broken." Preserve the transaction when either condition holds. This is a strict superset of both the pre-#3019 and post-#3019 behavior, so it cannot regress either issue. Verified against the reporter's repro on both the WaitHandle and V2 (channel) connection pools, and against the full manual TransactionTest suite (9/9 passing). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 * Docs | Add root cause analysis note for #4001 Records the delegated-root vs enlisted-participant distinction that this bug turns on, why #3019 swapped one case for the other rather than covering both, and why the union condition cannot reintroduce #2970. Also records the result of mutation testing the manual TransactionTest suite: the suite passes against the pre-#3019 condition (which carries #2970) and against the #3019 condition (which carries #4001), so it does not currently guard this line. The 9/9 pass rate is evidence of no collateral damage, not evidence that the fix works. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 * Docs | Trim RCA note for #4001 Drops the alternative-approach rationale, the residual-risk discussion, and the failed-reproduction-variants section, and renumbers the remainder. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 * Tests | Add regression tests for delegated transaction reset (#4001) Extract the reset-time transaction preservation predicate into an internal testable helper, ShouldPreserveTransactionOnReset, and pin its full truth table with unit tests. This also restores the Is2008OrNewer guard that the original one-line fix dropped. Preserving a delegated transaction root across a reset is only valid on SQL Server 2008 and newer; SQL Server 2005 is still an accepted TDS version, so the guard is load bearing. The tests were mutation tested against three buggy variants of the predicate -- the #3019 condition (reintroduces #4001), the pre-#3019 condition (reintroduces #2970), and the union without the 2008 guard -- and each one is caught. This follows the existing precedent of ResolveLoginTimeout / SqlConnectionInternalTimeoutTests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 * Docs | Correct and expand the RCA note for #4001 Fixes two factual errors flagged in review: - A delegated transaction root does not always have a null EnlistedTransaction. The property is set unconditionally on enlistment; null is specific to the transient half state left behind by DetachCurrentTransactionIfEnded. - Is2008OrNewer is not vestigial. SQL Server 2005 is still an accepted TDS version, so the guard can be false. Also documents the new regression tests, why an end-to-end reproduction was abandoned in favour of a helper level test, and removes an inaccurate claim that the new condition is a strict superset of the pre-#3019 one. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 * Tests | Make the reset predicate theory genuinely exhaustive The unpooled block enumerated only four of its eight input combinations while the doc comment claimed exhaustive coverage. Add the missing four so all sixteen combinations of the four booleans are pinned, and state the count explicitly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 * Docs | Hyphenate server-side in reset predicate docs Compound adjective. Comment-only change, no behavior impact. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 * Fix | Drop unsupported server-version guard from reset predicate The restored Is2008OrNewer guard reinstated only half of a retired safety mechanism. Pre-#3019, IsNonPoolableTransactionRoot both suppressed the preserve bit and routed the connection into pool stasis; being parked is what made the plain reset harmless. #3019 removed the property, and both pools now route on EnlistedTransaction alone, so a delegated root with a null EnlistedTransaction returns to the general pool. Suppressing the preserve bit without the stasis routing therefore reproduces #4001 on SQL Server 2005 -- which is below the supported floor of 2012 anyway. Predicate is now isPooled && (isTransactionRoot || hasEnlistedTransaction). Tests simplified from 19 to 11; both bug mutants still fail. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 * Tests | Remove duplicate reset predicate facts Keep the regression context beside the corresponding exhaustive theory rows instead of repeating those cases as separate facts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cadc8f9e-e4ac-4074-92ef-88e90df96091 --- .../4001-delegated-transaction-reset.md | 322 ++++++++++++++++++ .../Connection/SqlConnectionInternal.cs | 78 ++++- ...ConnectionInternalResetTransactionTests.cs | 81 +++++ 3 files changed, 476 insertions(+), 5 deletions(-) create mode 100644 doc/design-notes/4001-delegated-transaction-reset.md create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionInternalResetTransactionTests.cs diff --git a/doc/design-notes/4001-delegated-transaction-reset.md b/doc/design-notes/4001-delegated-transaction-reset.md new file mode 100644 index 0000000000..79d24fb04c --- /dev/null +++ b/doc/design-notes/4001-delegated-transaction-reset.md @@ -0,0 +1,322 @@ +# Root cause analysis: #4001 — pooled connection broken after `TransactionScope` rollback + +| | | +|---|---| +| **Issue** | [#4001](https://github.com/dotnet/SqlClient/issues/4001) | +| **Regressed by** | [#3019](https://github.com/dotnet/SqlClient/pull/3019) (`0322d44c7`), shipped in 6.1.0 | +| **Affected** | 6.1.0 → 6.1.6, `main` | +| **Last good** | 6.0.5 | +| **Code** | `SqlConnectionInternal.ResetConnection()` | + +This note records why the bug happens, why the previous fix caused it, why the new +condition cannot reintroduce the issue that fix was addressing, and — importantly — +what the existing test suite does and does not actually verify. + +--- + +## 1. Background: two ways a connection can be "in" a transaction + +This distinction is the crux of the entire bug. + +When a connection participates in a `TransactionScope`, it ends up in one of **two +distinct** states. They overlap, but neither implies the other. + +### Delegated root — "I *own* this transaction" + +Only one connection is involved, so `System.Transactions` delegates the transaction +down to SQL Server rather than paying for a distributed coordinator. The transaction +lives **on** the connection. + +- `IsTransactionRoot` → `true` +- `EnlistedTransaction` → set at first, then **cleared** later (see below) + +`IsTransactionRoot` is not a stored flag. It is derived: + +```csharp +internal bool IsTransactionRoot => DelegatedTransaction?.IsActive == true; +``` + +Enlistment sets `EnlistedTransaction` unconditionally, so a *freshly* delegated root +has both. But once the transaction is no longer `Active`, +`DbConnectionInternal.DetachCurrentTransactionIfEnded` clears `EnlistedTransaction`: + +```csharp +transactionIsDead = enlistedTransaction.TransactionInformation.Status != TransactionStatus.Active; +if (transactionIsDead) { DetachTransaction(enlistedTransaction, true); } +``` + +The delegated transaction, meanwhile, can still report `IsActive == true`. That +transient **half-state** — root, but no `EnlistedTransaction` — is precisely the state +issue #4001 reproduces in. + +### Enlisted participant — "I *joined* someone else's transaction" + +Multiple resources are involved, so a coordinator (MSDTC) owns the transaction and +each connection enlists in it. + +- `IsTransactionRoot` → `false` +- `EnlistedTransaction` → set + +### The trap + +Neither field subsumes the other: a delegated root can have a `null` +`EnlistedTransaction`, and an enlisted participant is never a root. Any check that +tests only one of these fields silently misses the other case — and does so with no +exception at the point of the mistake. + +--- + +## 2. Where the damage occurs + +When a connection is closed it returns to the pool and is **reset** — wiped clean for +the next consumer. If a transaction is still in flight, that reset must *preserve* it. +The entire decision is one boolean: + +```csharp +_parser.PrepareResetConnection(preserveTransaction); +``` + +Pass `false` while a transaction is genuinely live, and the TDS reset destroys the +server-side transaction **while `System.Transactions` still believes it exists**. + +The failure then surfaces later, some distance from the cause: + +1. `TransactionScope` disposes and rolls back. +2. `SqlDelegatedTransaction.Rollback` asks the server to roll back a transaction the + server no longer has. +3. The rollback fails, and SqlClient calls `DoomThisConnection()`. +4. The physical connection is now permanently marked broken. +5. With a small pool (the report used `MaxPoolSize=1`) that same doomed connection is + immediately handed back out. +6. The next caller gets: + +> `InvalidOperationException: The requested operation cannot be completed because the connection has been broken.` + +The exception names the connection, not the reset that ruined it. That distance +between cause and symptom is what makes this class of bug hard to trace. + +--- + +## 3. What PR #3019 actually changed + +The relevant diff from `0322d44c7`: + +```diff +- _parser.PrepareResetConnection(IsTransactionRoot && !IsNonPoolableTransactionRoot); ++ _parser.PrepareResetConnection(EnlistedTransaction is not null && Pool is not null); +``` + +The old helper was: + +```csharp +internal protected override bool IsNonPoolableTransactionRoot + => IsTransactionRoot && (!Is2008OrNewer || Pool == null); +``` + +Substituting, the pre-#3019 condition was: + +```csharp +IsTransactionRoot && Is2008OrNewer && Pool != null +``` + +The `Is2008OrNewer` term is **deliberately not carried forward**, for two reasons. + +First, it is outside the support matrix. The term is `false` for exactly one server +version — SQL Server 2005 — and the driver's supported floor is SQL Server 2012. + +Second, and more importantly, **it was never safe on its own.** `IsNonPoolableTransactionRoot` +had two jobs, not one. Besides suppressing the preserve bit, it also drove pool routing: +`DbConnectionPool` sent any connection it flagged into *stasis* rather than back into the +pool. Being parked is what made a plain reset harmless — the connection was never handed +to another caller. #3019 deleted the property entirely, and today's pools route on +`EnlistedTransaction` alone. A delegated root with a `null` `EnlistedTransaction` now goes +straight back to the **general** pool. + +Reinstating only the suppression half would therefore reset a live delegated transaction +*and* hand the connection out again — #4001 exactly, just narrowed to SQL Server 2005. +Half of a retired safety mechanism is worse than none of it. + +So the two conditions were: + +| | Question it asked | Covered | Missed | +|---|---|---|---| +| **Pre-#3019** | "Am I the *owner*?" | delegated root | enlisted → **#2970** | +| **#3019** | "Am I *enlisted*?" | enlisted | delegated root → **#4001** | + +**#3019 swapped one case for the other rather than covering both.** It genuinely fixed +#2970, and it traded it for #4001. Both conditions were half-right; neither was wrong +about the case it did cover. + +This reframes the fix: the goal is not to undo #3019, it is to finish it. + +--- + +## 4. The fix + +The predicate is extracted into a helper so it can be tested directly: + +```csharp +internal static bool ShouldPreserveTransactionOnReset( + bool isPooled, + bool isTransactionRoot, + bool hasEnlistedTransaction) +{ + if (!isPooled) + { + return false; + } + + return isTransactionRoot || hasEnlistedTransaction; +} +``` + +This is `OLD || NEW`: each arm answers one of the two questions from section 1, so the +predicate is `true` wherever either predecessor was. + +--- + +## 5. Why this cannot reintroduce #2970 + +This is the question that matters most, and it is answerable by inspection rather than +by testing. Every reachable state, for a pooled connection: + +| `IsTransactionRoot` | `EnlistedTransaction` | Pre-#3019 | #3019 | **This fix** | +|:---:|:---:|:---:|:---:|:---:| +| `false` | `null` | `false` | `false` | `false` | +| **`true`** | **`null`** | ✅ `true` | ❌ `false` ← **#4001** | ✅ **`true`** | +| **`false`** | **set** | ❌ `false` ← **#2970** | ✅ `true` | ✅ **`true`** | +| `true` | set | `true` | `true` | `true` | + +Read the **#2970 row**. That is the row PR #3019 was created to fix, and this fix still +evaluates `true` there. It is untouched. + +Reintroducing #2970 would require that cell to flip to `false`, and `A || B` cannot +evaluate `false` while `B` is `true`. The guarantee is structural, not empirical. + +Because each arm reproduces its original predecessor exactly, the condition returns +`true` wherever either predecessor did, and never returns `false` where one of them +returned `true`. + +--- + +## 6. How the root cause was established + +The cause was **proven at runtime, not inferred**. The driver was temporarily +instrumented at the reset site and at `DoomThisConnection()`. The captured state at the +critical reset: + +``` +[RESET] obj=4 preserve=False root=False deleg=null enlisted=null pool=set +[RESET] obj=7 preserve=False root=False deleg=null enlisted=null pool=set +[RESET] obj=7 preserve=False root=True deleg=active=True enlisted=null pool=set <-- old: true, new: false +[DOOM] obj=7 + at Microsoft.Data.SqlClient.SqlDelegatedTransaction.Rollback(...) + at System.Transactions.Transaction.Rollback() + at System.Transactions.TransactionScope.InternalDispose() + at System.Transactions.TransactionScope.Dispose() +[FAIL] Bug reproduced +``` + +The third reset is the bug caught in the act: `root=True`, `deleg.IsActive=True`, +`enlisted=null`, and `preserve=False`. A live delegated transaction being discarded. + +This mattered, because **the initial hypothesis was wrong.** The first theory was that +#3019 had made the condition *too broad*, and a narrowing fix was written on that +basis. It did not work. The instrumentation showed the opposite — #3019 had *narrowed* +the condition, not widened it — and the fix was rewritten accordingly. Without runtime +evidence this would have been fixed in the wrong direction. + +All instrumentation was removed before commit. + +--- + +## 7. What the existing tests actually verify + +This section is deliberately blunt, because the intuitive answer is wrong. + +### Bisection + +Against the reporter's reproduction (NHibernate 5.5.2, `MaxPoolSize=1`, +`TransactionScope` with a failed DTC promotion): + +**6.0.5 ✅ · 6.1.0 ❌ · 6.1.1 ❌ · 6.1.4 ❌ · `main` ❌** + +This places the regression in the 6.1.0 window, consistent with #3019. + +### Both pool implementations, both directions + +| | without fix | with fix | +|---|---|---| +| `WaitHandleDbConnectionPool` (default) | ❌ reproduces | ✅ passes | +| `ChannelDbConnectionPool` (`UseConnectionPoolV2`) | ❌ reproduces | ✅ passes | + +The **left-hand column is the load-bearing one.** It was produced by stashing the fix +and rebuilding. Without it, a pass on the V2 pool could simply mean V2 never reaches +this code path, which would prove nothing. + +(V2 in released 6.1.4 throws `NotImplementedException`, so only `main` was testable.) + +### Mutation testing of the manual suite + +`--filter "FullyQualifiedName~TransactionTest"` reports **9/9 passing** with the fix. +That number is easy to over-read, so the suite was mutation-tested: the condition was +replaced with each known-buggy variant and the suite re-run. + +| Condition compiled in | Bug it contains | Suite result | +|---|---|---| +| Pre-#3019 (`IsTransactionRoot && Pool is not null`) | **#2970** | **9/9 passed** | +| #3019 (`EnlistedTransaction is not null && Pool is not null`) | **#4001** | **9/9 passed** | +| This fix (union) | none | 9/9 passed | + +**The suite passes on all three.** It does not detect either bug, and therefore does +not guard this line at all in this environment. + +`Test_EnlistedTransactionPreservedWhilePooled` — the test added by #3019 specifically +to cover #2970 — is tagged `[Trait("Category", "flaky")]` and passes against code that +carries the #2970 bug. + +The practical conclusion: **the 9/9 result is evidence of no collateral damage, not +evidence that the fix works.** + +### The regression test that was added + +An end-to-end reproduction was attempted extensively and abandoned. The `#4001` state +requires a narrow simultaneity — the transaction's status already non-`Active` (so +`DetachCurrentTransactionIfEnded` has cleared `EnlistedTransaction`) while +`DelegatedTransaction.IsActive` is still `true` — and which of two teardown paths in +`SqlDelegatedTransaction` wins is a race: + +- `TransactionEnded` sets `_active = false` and *immediately* calls + `DoomThisConnection()`. If this path runs first, the delegate is already inactive + before any reset, so the state is never observed. +- `Rollback`, driven from `TransactionScope.Dispose`, sets `_active = false` *after* the + reset. This is the ordering the reporter hit. + +Roughly twenty harness variants — varying pool size, pool implementation, promotion +success, explicit rollback, and parking the delegate in the transacted pool ahead of the +enlistment — consistently drove the first path. A test built on that race would be +flaky, which is the same defect `Test_EnlistedTransactionPreservedWhilePooled` already +demonstrates. + +Instead the predicate was extracted into +`SqlConnectionInternal.ShouldPreserveTransactionOnReset` and pinned directly by +`SqlConnectionInternalResetTransactionTests`, following the existing precedent of +`ResolveLoginTimeout` / `SqlConnectionInternalTimeoutTests`. The tests were themselves +mutation-tested: + +| Condition compiled into the helper | Bug it contains | New tests | +|---|---|---| +| `hasEnlistedTransaction` (#3019) | **#4001** | ❌ 1 failed | +| `isTransactionRoot` (pre-#3019) | **#2970** | ❌ 1 failed | +| The shipped condition | none | ✅ 8 passed | + +This satisfies "fails before the change, passes after" for **both** regressions, and +does so deterministically and without a server. + +--- + +## 8. Related + +- **#2970** — the issue #3019 was fixing. Fully preserved by this change (section 5). +- **#2285** — reports the same exception with no reproduction. Plausibly the same root + cause, though unconfirmed. diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs index ec210e407a..fdf6f07f70 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs @@ -3973,11 +3973,14 @@ internal override void ResetConnection() if (_fResetConnection) { - // Pooled connections that are enlisted in a transaction must have their transaction - // preserved when resetting the connection state. Otherwise, future uses of the connection - // from the pool will execute outside the transaction, in auto-commit mode. - // https://github.com/dotnet/SqlClient/issues/2970 - _parser.PrepareResetConnection(EnlistedTransaction is not null && Pool is not null); + // Pooled connections that are tied to a transaction must have that transaction + // preserved when resetting the connection state. Otherwise, future uses of the + // connection from the pool will execute outside the transaction, in auto-commit + // mode. See ShouldPreserveTransactionOnReset for the full rationale. + _parser.PrepareResetConnection(ShouldPreserveTransactionOnReset( + isPooled: Pool is not null, + isTransactionRoot: IsTransactionRoot, + hasEnlistedTransaction: EnlistedTransaction is not null)); // Reset dictionary values, since calling reset will not send us env_changes. CurrentDatabase = _originalDatabase; @@ -3985,6 +3988,71 @@ internal override void ResetConnection() } } + /// + /// Decides whether a connection reset must preserve the server-side transaction, i.e. + /// whether ST_RESET_CONNECTION_PRESERVE_TRANSACTION should be sent instead of a + /// plain ST_RESET_CONNECTION. + /// + /// Whether this connection belongs to a pool. + /// + /// Whether this connection is the root of a delegated transaction, i.e. + /// . + /// + /// + /// Whether this connection has a non-null . + /// + /// + /// + /// A pooled connection can be tied to a transaction in two independent ways, and both + /// must be preserved across a reset: + /// + /// + /// + /// It is the root of a delegated transaction: the transaction was delegated to + /// this connection, so it lives on the server session this connection owns. Missing this + /// case resets the server-side transaction out from under System.Transactions, which + /// later breaks the connection while it is being recycled through the pool. + /// See https://github.com/dotnet/SqlClient/issues/4001. + /// + /// + /// It has enlisted in a transaction, so EnlistedTransaction is set. Missing + /// this case causes subsequent uses of the connection to run outside the transaction, in + /// auto-commit mode. See https://github.com/dotnet/SqlClient/issues/2970. + /// + /// + /// + /// These two conditions overlap but neither implies the other. A freshly delegated root + /// has both flags set, because enlistment sets EnlistedTransaction unconditionally. + /// However, once the transaction is no longer Active, + /// DetachCurrentTransactionIfEnded clears EnlistedTransaction while the + /// delegated transaction can still report itself as active. That transient half-state is + /// root-only, and it is exactly the state issue #4001 reproduces in. + /// + /// + /// There is deliberately no server-version guard here. Before + /// https://github.com/dotnet/SqlClient/pull/3019, a delegated root on a pre-2008 server + /// was excluded via IsNonPoolableTransactionRoot, but that property also routed + /// such connections into stasis rather than back into the pool, which is what made + /// suppressing the preserve bit safe. #3019 removed the property, and neither pool puts a + /// poolable root into stasis today, so a version guard would now return the connection to + /// the general pool with its transaction silently reset. That is issue #4001 again. + /// SQL Server 2005 is also outside the supported matrix. + /// + /// + internal static bool ShouldPreserveTransactionOnReset( + bool isPooled, + bool isTransactionRoot, + bool hasEnlistedTransaction) + { + // A connection with no pool is not recycled, so there is nothing to preserve for. + if (!isPooled) + { + return false; + } + + return isTransactionRoot || hasEnlistedTransaction; + } + private void ResolveExtendedServerName(ServerInfo serverInfo, bool aliasLookup, SqlConnectionOptions options) { // @TODO: Invert to save on indentation diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionInternalResetTransactionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionInternalResetTransactionTests.cs new file mode 100644 index 0000000000..dcf7262e78 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionInternalResetTransactionTests.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Data.SqlClient.Connection; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests; + +/// +/// Verifies , which decides +/// whether a pooled connection is reset with ST_RESET_CONNECTION_PRESERVE_TRANSACTION +/// rather than a plain ST_RESET_CONNECTION. +/// +/// Getting this predicate wrong has caused two separate regressions, in opposite directions: +/// +/// +/// Testing only IsTransactionRoot misses connections that merely enlisted in someone +/// else's transaction, so they are reset and subsequently run in auto-commit mode +/// (https://github.com/dotnet/SqlClient/issues/2970). +/// +/// +/// Testing only EnlistedTransaction misses delegated transaction roots, whose +/// server-side transaction is then reset out from under System.Transactions, corrupting the +/// connection as it is recycled through the pool +/// (https://github.com/dotnet/SqlClient/issues/4001). +/// +/// +/// +/// Both conditions must therefore be honored. Asserting against the extracted predicate covers +/// every combination deterministically, including state combinations that are transient and +/// racy to stage against a live server. +/// +public class SqlConnectionInternalResetTransactionTests +{ + /// + /// Exhaustively pins the predicate over all eight combinations of its three boolean inputs. + /// + /// The expectations encode two rules: + /// + /// An unpooled connection is never recycled, so nothing is ever preserved. + /// A pooled connection tied to a transaction in either way must be preserved. + /// + /// + [Theory] + // An unpooled connection is destroyed rather than recycled, so there is no subsequent use to + // protect and its transaction must not be preserved, regardless of any other state. + [InlineData(false, false, false, false)] + [InlineData(false, false, true, false)] + [InlineData(false, true, false, false)] + [InlineData(false, true, true, false)] + // Pooled, no transaction of any kind: nothing to preserve. + [InlineData(true, false, false, false)] + // Regression guard for #2970: an implementation keyed only on IsTransactionRoot returns false + // for a pooled connection enlisted in a transaction it does not own. + [InlineData(true, false, true, true)] + // Regression guard for #4001: an implementation keyed only on EnlistedTransaction returns + // false in this transient half-state, after the enlistment is detached but while the delegated + // transaction still reports itself as active. + [InlineData(true, true, false, true)] + // Pooled connection that is both a delegated root and has an EnlistedTransaction. This is + // the common state immediately after enlistment, since enlistment sets EnlistedTransaction + // unconditionally. + [InlineData(true, true, true, true)] + public void ShouldPreserveTransactionOnReset_CoversBothTransactionOwnershipModes( + bool isPooled, + bool isTransactionRoot, + bool hasEnlistedTransaction, + bool expected) + { + // Act + bool actual = SqlConnectionInternal.ShouldPreserveTransactionOnReset( + isPooled: isPooled, + isTransactionRoot: isTransactionRoot, + hasEnlistedTransaction: hasEnlistedTransaction); + + // Assert + Assert.Equal(expected, actual); + } + +} From 9e42a9a60ccb8783cfcc2eb754b9252ae33c4c93 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:57:11 -0300 Subject: [PATCH 32/51] Pipelines | Validate every package the OneBranch build produces (#4655) * Derive APIScan versions from package versions * Pipelines | Pre-compute all OneBranch package and file versions Make the compute-versions stage the single source of every version the OneBranch build jobs consume, so nothing is re-derived downstream. - Remove the addRevision mode entirely. Package versions now have a single shape driven by the pipeline build number, and the 16-bit revision wrapping, the four-part package base handling, and the Build.BuildId plumbing are gone. - Move package version stamping out of PowerShell and into Versions.props. BuildSuffix now does what it always documented: it turns a stable base into a prerelease. Any version carrying a prerelease tag, from either source, is stamped with the build number; released versions are left untouched. - Publish SqlClient and SqlServer file versions from the compute-versions stage and pass them into the build jobs, which previously received a raw build number and re-derived the file version through MSBuild. build.proj gains opt-in FileVersion* arguments, so PR/CI and local builds are unchanged. - Fix SBOM metadata, which reported the pipeline run number as the version of a single hardcoded package name. Each build job now supplies the name and computed version of the package it produces, and jobs that publish no packages disable SBOM generation instead. * Pipelines | Validate every package the OneBranch build produces Re-enable the package validation stage, which had been commented out pending the version pre-computation that landed in the previous change, and widen it from one package to all six. - Add a package_validation stage that depends on all four build stages. Every package is downloaded into one tree and validated together so PackageValidator can apply its cross-package rules: the SqlClient family must share a single version, and their inter-package dependency ranges must agree. Validating one package at a time would silently skip all of those findings. - Assert the versions the compute-versions stage already published, rather than re-deriving them. The family version is applied as a wildcard expectation, so a mismatch in any one package is caught along with the case where every package is consistently wrong; Microsoft.SqlServer.Server overrides it by id. When SqlServer is not built its expectations are omitted entirely, because the validator rejects an expectation whose value is empty. - Gate on error and missing-symbols always, plus package-unsigned on official runs. missing-symbols is a warning and package-unsigned is info, so neither is covered by the error severity and both must be named explicitly. Non-official runs are deliberately unsigned, so gating them on package-unsigned would always fail. - Make the release stage depend on package validation, so a package that fails validation is never published. - Replace the SqlClient-only validate-signed-package-job, which checked one package, could not detect missing files, indexed extracted content positionally, and depended on a hardcoded sn.exe path. Authenticode and NuGet signature verification now cover every produced package. Step and job logic lives in scripts with Pester coverage rather than inline YAML, matching compute-versions and publish-symbols. * Pipelines | Disable Roslyn SDL analysis in the package validation job 1ES auto-injects the RoslynAnalyzers task into any job containing a DotNetCoreCLI build task, and drives the build itself. The validation job's only compile is PackageValidator, a build-time tool that never ships, and the injected run is launched from the host where the container's dotnet does not exist, so it failed with exit code 17 and failed the job even though package validation passed. * Pipelines | Fail fast on validator errors and normalize FailOn tokens Addresses review feedback on PR #4655. The JSON-report invocation of PackageValidator ignored its exit code, so a validator crash produced a misleading report artifact and let the gated run obscure the real cause. Capture and check the code before writing the success message or reaching the gate. FailOn arrives from the pipeline as a single comma-joined token, which PowerShell -File argument mode does not split into an array. Normalize the parameter by splitting, trimming, and dropping empties, and quote the YAML argument so the value is one token in every invocation mode. * Pipelines | Gate package validation on dependency and strong-name findings Addresses review feedback on PR #4655. The error severity covers only error-severity findings, so the warning and info categories this job exists to catch were slipping through the gate. dependency-inconsistency is a warning, and mismatched family dependency ranges are precisely what validating the whole drop at once is meant to find, so gate it on every run. delay-signed is a warning and unsigned is info. Non-official builds have no access to the real strong-name key and are delay-signed by design, so gate those two on official runs only, alongside package-unsigned. * Pipelines | Gate strong-name findings on every run, not just official Addresses review feedback on PR #4655. The previous split assumed non-official builds cannot strong-name sign, but build-buildproj-step.yml downloads netfxKeypair.snk and passes SigningKeyPath unconditionally, so every OneBranch build signs with the real key. Run 26251.2 confirms it: 59 of 59 implementation assemblies reported Signed on a non-official run, with no delay-signed or unsigned findings. Gating delay-signed and unsigned only on official runs therefore left the one signing type the pipeline always applies unvalidated on PR builds, where a regression that drops the key would go unnoticed. Gate both everywhere. package-unsigned stays official-only. NuGet package signing is an ESRP step condition on shouldSignPackage, so non-official packages genuinely carry no .signature.p7s. * Pipelines | Clarify that the reporting validator run is ungated, not infallible Addresses PR #4655 review feedback: the first PackageValidator invocation is ungated so the JSON report survives a failing run, but it still fails the step when the validator itself errors. --- .../onebranch-pipeline-design.instructions.md | 17 +- .../onebranch/jobs/validate-packages-job.yml | 206 +++++++++ .../jobs/validate-signed-package-job.yml | 395 ------------------ .../onebranch/scripts/tests/README.md | 6 + .../scripts/tests/validate-packages.Tests.ps1 | 188 +++++++++ .../verify-assembly-signatures.Tests.ps1 | 149 +++++++ .../tests/verify-package-signatures.Tests.ps1 | 87 ++++ .../onebranch/scripts/validate-packages.ps1 | 213 ++++++++++ .../scripts/verify-assembly-signatures.ps1 | 99 +++++ .../scripts/verify-package-signatures.ps1 | 72 ++++ .../onebranch/stages/build-stages.yml | 53 ++- .../onebranch/stages/release-stages.yml | 4 + .../steps/validate-packages-step.yml | 80 ++++ 13 files changed, 1159 insertions(+), 410 deletions(-) create mode 100644 eng/pipelines/onebranch/jobs/validate-packages-job.yml delete mode 100644 eng/pipelines/onebranch/jobs/validate-signed-package-job.yml create mode 100644 eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 create mode 100644 eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 create mode 100644 eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 create mode 100644 eng/pipelines/onebranch/scripts/validate-packages.ps1 create mode 100644 eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 create mode 100644 eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 create mode 100644 eng/pipelines/onebranch/steps/validate-packages-step.yml diff --git a/.github/instructions/onebranch-pipeline-design.instructions.md b/.github/instructions/onebranch-pipeline-design.instructions.md index 7a24fd3799..75ea6ea303 100644 --- a/.github/instructions/onebranch-pipeline-design.instructions.md +++ b/.github/instructions/onebranch-pipeline-design.instructions.md @@ -34,7 +34,7 @@ Defined in `stages/build-stages.yml`. Four build stages plus validation, ordered - **`build_abstractions`** (Stage 2) — Abstractions; `dependsOn: build_independent`; downloads Logging artifact - **`build_dependent`** (Stage 3) — SqlClient and Extensions.Azure in parallel; `dependsOn: build_abstractions`; downloads Abstractions + Logging artifacts - **`build_addons`** (Stage 4) — AKV Provider; `dependsOn: build_dependent`; downloads SqlClient + Abstractions + Logging artifacts -- **`sqlclient_package_validation`** — Validates signed SqlClient package; `dependsOn: build_dependent`; runs in parallel with Stage 4 +- **`package_validation`** (Stage 5) — Validates every package produced by the run; `dependsOn` all four build stages plus `compute_versions` Each build job copies PDB files into `$(JOB_OUTPUT)/symbols/` so they are included in the auto-published pipeline artifact alongside the NuGet packages in `$(JOB_OUTPUT)/packages/`. @@ -46,7 +46,7 @@ Stage conditional rules: ## Job Templates - **`build-buildproj-job.yml`** — Shared build.proj-driven package job used for all shipped packages. Flow: build via `build.proj` → optional ESRP DLL signing → pack via `build.proj` → optional ESRP NuGet signing → copy outputs for APIScan/artifacts -- **`validate-signed-package-job.yml`** — Validates signed MDS package (signature, strong names, folder structure, target frameworks) +- **`validate-packages-job.yml`** — Validates every package produced by the run. Downloads all package artifacts into one tree and validates them together, so `tools/PackageValidator` can apply its cross-package rules (the SqlClient family must share one version, and inter-package dependency ranges must agree); validating per package would silently skip those findings. Runs on Windows because Authenticode verification has no Linux equivalent - **`publish-nuget-package-job.yml`** — Reusable release job using OneBranch `templateContext.type: releaseJob` with `inputs` for artifact download; pushes via `NuGetCommand@2` - **`publish-symbols-job.yml`** — Reusable symbols job: downloads a build artifact, locates PDBs under `symbols/`, and invokes `publish-symbols-step.yml` @@ -56,6 +56,19 @@ When adding a new package to the OneBranch flow: - Add version variables to `variables/common-variables.yml` - Add artifact name variables to `variables/onebranch-variables.yml` +## Package Validation Stage + +- Defined in `stages/build-stages.yml`; produces stage `package_validation` +- Consumes the package and file versions published by `compute_versions` and asserts the produced packages carry exactly those values, so nothing is re-derived +- All packages are validated together in one job so `tools/PackageValidator` can apply cross-package rules; the SqlServer artifact and its expectations are conditional on `buildSqlServer` +- Expectations use the validator's `[id=]value` form: the SqlClient family version is applied as a wildcard (proving the family agrees, and catching the case where all packages are consistently wrong), with `Microsoft.SqlServer.Server` as a per-id override +- When SqlServer is not built its expectations are **omitted entirely** rather than passed empty — the validator rejects an expectation with an empty value +- Gate categories are derived from `isOfficial`: `error`, `missing-symbols`, `dependency-inconsistency`, `delay-signed`, and `unsigned` always, plus `package-unsigned` on official runs only. The `error` severity covers only error-severity findings, so each warning/info category must be named explicitly — `missing-symbols`, `dependency-inconsistency`, and `delay-signed` are warnings, and `unsigned` and `package-unsigned` are info. Strong-name signing is unconditional in `build-buildproj-step.yml`, so the two strong-name categories gate everywhere; NuGet package signing is ESRP and official-only, so `package-unsigned` would fire on every non-official run +- The validator runs twice: once with `--json` and no gate so the report exists even for a failing run, then once human-readable with the gate so failures appear in the job log +- Signature verification (`dotnet nuget verify --all`, Authenticode) runs on official builds only, and verifies that signatures are *trusted* — PackageValidator reports only their presence, from metadata +- The release stage `dependsOn: package_validation`, so a package that fails validation is never published +- Step and job logic lives in `scripts/validate-packages.ps1`, `scripts/verify-package-signatures.ps1`, and `scripts/verify-assembly-signatures.ps1`, each with Pester tests under `scripts/tests/` + ## Symbols Publishing Stage - Defined in `stages/publish-symbols-stage.yml`; produces stage `publish_symbols` diff --git a/eng/pipelines/onebranch/jobs/validate-packages-job.yml b/eng/pipelines/onebranch/jobs/validate-packages-job.yml new file mode 100644 index 0000000000..f1501c7d1d --- /dev/null +++ b/eng/pipelines/onebranch/jobs/validate-packages-job.yml @@ -0,0 +1,206 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Validates every NuGet package produced by this run. +# +# All packages are downloaded into a single tree and validated together in one job, rather than one +# job per package, so that PackageValidator can apply its cross-package rules: the SqlClient family +# must share a single version, and their inter-package dependency ranges must agree. +# +# The job runs on Windows because Authenticode verification has no equivalent on the Linux agents. +# PackageValidator itself is cross-platform, so only the signature checks are OS-bound. + +parameters: + # Package Parameters ----------------------------------------------------- + + - name: abstractionsArtifactsName + type: string + + - name: akvProviderArtifactsName + type: string + + - name: azureArtifactsName + type: string + + - name: loggingArtifactsName + type: string + + - name: sqlClientArtifactsName + type: string + + - name: sqlServerArtifactsName + type: string + + # Version Parameters ----------------------------------------------------- + # Pre-computed by the compute-versions stage. Validation asserts the produced packages carry + # exactly these versions, so nothing here is re-derived. + + - name: sqlClientPackageVersion + type: string + + - name: sqlClientFileVersion + type: string + + - name: sqlServerPackageVersion + type: string + + - name: sqlServerFileVersion + type: string + + # Behaviour Parameters --------------------------------------------------- + + # Whether Microsoft.SqlServer.Server was built this run. When false there is no SqlServer + # artifact to download and no SqlServer package in the drop to validate. + - name: buildSqlServer + type: boolean + + # True for official builds, which sign their packages and assemblies. Signature verification is + # skipped otherwise, because non-official runs deliberately produce unsigned output. + - name: isOfficial + type: boolean + +jobs: + - job: validate_packages + displayName: 'Validate Packages' + + pool: + type: windows + + # 1ES auto-injects Roslyn into any job holding a DotNetCoreCLI build task, which here is only + # the PackageValidator tool build -- never shipped, so out of SDL scope. + templateContext: + sdl: + roslyn: + enabled: false + + variables: + - name: ob_outputDirectory + value: '$(JOB_OUTPUT)' + + # This job inspects already-built packages and produces no assemblies, so it has nothing for + # APIScan or BinSkim to scan and no shipping component to describe in an SBOM. The build + # jobs cover all three for the packages they produce. + - name: ob_sdl_apiscan_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_sbom_enabled + value: false + + # Every package artifact is downloaded beneath this root, each into its own subdirectory so + # that identically-named files from different packages cannot collide. + - name: packagesRoot + value: '$(Pipeline.Workspace)/validate-packages' + + - name: extractRoot + value: '$(Pipeline.Workspace)/validate-extract' + + steps: + - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self + + # Only the packages themselves are needed, not the full build output each artifact carries. + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Logging' + inputs: + artifactName: '${{ parameters.loggingArtifactsName }}' + targetPath: '$(packagesRoot)/Logging' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Abstractions' + inputs: + artifactName: '${{ parameters.abstractionsArtifactsName }}' + targetPath: '$(packagesRoot)/Abstractions' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - SqlClient' + inputs: + artifactName: '${{ parameters.sqlClientArtifactsName }}' + targetPath: '$(packagesRoot)/SqlClient' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Azure' + inputs: + artifactName: '${{ parameters.azureArtifactsName }}' + targetPath: '$(packagesRoot)/Azure' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - AkvProvider' + inputs: + artifactName: '${{ parameters.akvProviderArtifactsName }}' + targetPath: '$(packagesRoot)/AkvProvider' + patterns: '**/*.*nupkg' + + - ${{ if eq(parameters.buildSqlServer, true) }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - SqlServer' + inputs: + artifactName: '${{ parameters.sqlServerArtifactsName }}' + targetPath: '$(packagesRoot)/SqlServer' + patterns: '**/*.*nupkg' + + # PackageValidator targets net10.0, which the repo's global.json already pins. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + + - template: /eng/pipelines/onebranch/steps/validate-packages-step.yml@self + parameters: + packagesPath: '$(packagesRoot)' + reportPath: '$(JOB_OUTPUT)/validation/package-validation.json' + sqlClientPackageVersion: '${{ parameters.sqlClientPackageVersion }}' + sqlClientFileVersion: '${{ parameters.sqlClientFileVersion }}' + # Omitted when SqlServer is not built: its package is absent from the drop, and the + # validator rejects an expectation with an empty value. + ${{ if eq(parameters.buildSqlServer, true) }}: + sqlServerPackageVersion: '${{ parameters.sqlServerPackageVersion }}' + sqlServerFileVersion: '${{ parameters.sqlServerFileVersion }}' + # The error severity covers only error-severity findings, so every warning/info category + # this job relies on must be named explicitly: missing-symbols, dependency-inconsistency + # and delay-signed are warnings, and unsigned and package-unsigned are info. + # + # Strong-name signing is unconditional in build-buildproj-step.yml, so delay-signed and + # unsigned gate on every run. NuGet package signing is ESRP and runs on official builds + # only, so package-unsigned would fire on every non-official build. + ${{ if eq(parameters.isOfficial, true) }}: + failOn: + - error + - missing-symbols + - dependency-inconsistency + - delay-signed + - unsigned + - package-unsigned + ${{ else }}: + failOn: + - error + - missing-symbols + - dependency-inconsistency + - delay-signed + - unsigned + + # Signature verification, official builds only. PackageValidator reports strong-name and + # NuGet signature *presence* cross-platform; these steps additionally verify that the + # signatures are trusted, which requires the Windows trust store. + - ${{ if eq(parameters.isOfficial, true) }}: + - task: PowerShell@2 + displayName: 'Verify NuGet package signatures' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 + arguments: >- + -PackagesPath "$(packagesRoot)" + + - task: PowerShell@2 + displayName: 'Verify assembly Authenticode signatures' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 + arguments: >- + -PackagesPath "$(packagesRoot)" + -ExtractPath "$(extractRoot)" diff --git a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml b/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml deleted file mode 100644 index 818b902cf7..0000000000 --- a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml +++ /dev/null @@ -1,395 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# -parameters: - # The name of the pipeline artifact to download that contains the SqlClient NuGet package. - - name: artifactName - type: string - - # List of versions of dotnet that are *allowed to exist* in the NuGet package. Separators do not - # matter as the folders in lib, runtime, etc are simply checked to see if they exist in this - # string. - - name: expectedDotnetVersions - type: string - default: 'net462;net8.0;net9.0;netstandard2.0' - - # Expected file version of the assemblies within the package. This should be of the form: - # (major).(minor).(patch).(buildNumber) - - name: expectedFileVersion - type: string - - # List of folders that are *allowed to exist* in the NuGet package. Separators do not matter as - # the folders are simply checked to see if they exist in this string. - - name: expectedFolderNames - type: string - default: 'lib;ref;runtimes' - - # Expected NuGet package version. Used to build the installation path. This should be of the - # form: (major).(minor).(patch)[-preview(preview_number)] - - name: expectedPackageVersion - type: string - - # True if this build is an official build. This will be used to gate some checks - # that only apply to official builds, such as signature verification. - - name: isOfficial - type: boolean - -jobs: - - job: validate_nuget_package - displayName: "Validate NuGet package" - - pool: - type: windows # read more about custom job pool types at https://aka.ms/obpipelines/yaml/jobs - isCustom: true - name: ADO-1ES-Pool - vmImage: "ADO-MMS22-SQL19" - - variables: # More settings at https://aka.ms/obpipelines/yaml/jobs - - # This job installs and inspects an already-built package rather than producing assemblies, - # so it has no APIScan software name/version to report. The build jobs scan those assemblies. - - name: ob_sdl_apiscan_enabled - value: false - - # Likewise it produces no package, and sets no sbomPackage* values for globalSdl to resolve. - - name: ob_sdl_sbom_enabled - value: false - - # Path within the downloaded artifact where NuGet packages are located. - - name: artifactPath - value: '$(Pipeline.Workspace)\${{ parameters.artifactName }}' - - # Path to the SqlClient NuGet package after installation. This path will only exist once the package - # been installed. - - name: nugetPackageInstallPath - value: '$(Pipeline.Workspace)\nugetPackageInstalls\Microsoft.Data.SqlClient.${{ parameters.expectedPackageVersion }}' - - # Root folder where NuGet package will be installed locally - - name: nugetPackageInstallRoot - value: '$(Pipeline.Workspace)\nugetPackageInstalls' - - steps: - - template: '/eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self' - - - task: NuGetToolInstaller@1 - displayName: "Install NuGet" - - - powershell: | - echo "> 1. List all local cache directory paths" - nuget locals all -List - - echo "> 2. Clear all files from local cache directories" - nuget locals all -Clear - displayName: "Clear local cache" - - # Download NuGet packages from the specified build artifact. - - download: current - artifact: ${{ parameters.artifactName }} - patterns: "**/*.*nupkg" - displayName: "Download NuGet Package" - - # Verify secure signatures on the NuGet packages. - # NOTE: Packages will only be signed if the build is official. - - ${{ if eq(parameters.isOfficial, true) }}: - - powershell: | - # Propagate parameters to PS variables ####################### - $artifactPath = "${{ variables.artifactPath }}" - echo "artifactPath= $artifactPath" - - # Discover packages ########################################## - $packageFiles = Get-ChildItem -Path $artifactPath -Recurse -File -Include *.nupkg,*.snupkg - if ($packageFiles.Count -eq 0) - { - Write-Error "No NuGet package files were found under '$artifactPath'." - } - - # Verify package signatures ################################## - echo "> 1. Verify signature of source package(s)" - $packageFiles | Where-Object { $_.Extension -eq ".nupkg" } | ForEach-Object { - nuget verify -All $_.FullName - } - - echo "> 2. Verify signature of symbols package(s)" - $packageFiles | Where-Object { $_.Extension -eq ".snupkg" } | ForEach-Object { - nuget verify -All $_.FullName - } - displayName: "Verify nuget signature" - - # Install NuGet package to the temporary directory - - powershell: | - # Propagate pipeline to PS variables ############################# - $artifactPath = "${{ variables.artifactPath }}" - echo "artifactPath= $artifactPath" - - $expectedPackageVersion = "${{ parameters.expectedPackageVersion }}" - echo "expectedPackageVersion= $expectedPackageVersion" - - $nugetPackageInstallRoot = "${{ variables.nugetPackageInstallRoot }}" - echo "nugetPackageInstallRoot= $nugetPackageInstallRoot" - - # Find the SqlClient NuGet package ############################### - Get-ChildItem "$artifactPath" -Recurse - - $packagePaths = @(Get-ChildItem -Path $artifactPath -Recurse -File -Filter "Microsoft.Data.SqlClient.$expectedPackageVersion.nupkg") - if ($packagePaths.Count -eq 0) - { - Write-Error "Unable to find Microsoft.Data.SqlClient.$expectedPackageVersion.nupkg under '$artifactPath'." - } - if ($packagePaths.Count -gt 1) - { - Write-Error "Multiple Microsoft.Data.SqlClient.$expectedPackageVersion.nupkg files were found under '$artifactPath'." - } - - $packageSource = Split-Path -Path $packagePaths[0].FullName -Parent - echo "Found package path: $($packagePaths[0].FullName)" - echo "Using package source: $packageSource" - - # Install NuGet Package ########################################## - echo "> 1. Installing Microsoft.Data.SqlClient NuGet package..." - Install-Package ` - -Name "Microsoft.Data.SqlClient" ` - -Source "$packageSource" ` - -Destination $nugetPackageInstallRoot ` - -Force ` - -SkipDependencies - - echo "> 2. Listing contents of installed Microsoft.Data.SqlClient NuGet package:" - Write-Host $nugetPackageInstallRoot - Get-ChildItem $nugetPackageInstallRoot - displayName: "Install NuGet Package" - - # Find all DLL files in the installed NuGet package, verify each is signed with a strong name - - powershell: | - # Propagate pipeline to PS variables ############################# - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify strong name signing ##################################### - echo "> 1. Verifying strong name signing of DLLs ..." - - # @TODO: This path seems brittle to VS upgrades, can we make it more flexible? - $snPath = "C:\Program Files (x86)\Microsoft SDKs\Windows\*\bin\NETFX 4.8.1 Tools\sn.exe" - - $dllFiles = Get-ChildItem -Path $nugetPackageInstallPath -Recurse -Filter *.dll - $badDlls = @() - foreach ($file in $dllFiles) - { - # Run sn.exe to verify the strong name on each dll - $result = & $snPath -vf $file.FullName - Write-OutPut $result - - # if the dll is not valid, it would be delay signed or test-signed which is not meant for production - if($result[$result.Length-1] -notlike "* is valid") - { - $badDlls += $result[$result.Length-1] - } - } - if($badDlls.Count -gt 0) - { - Write-OutPut "Error: Invalid dlls are detected. Check the list below:" - foreach($dll in $badDlls) - { - Write-Output $dll - } - Exit -1 - } - Write-Host "Strong name has been verified for all dlls" - displayName: "Verify assembly strong names" - - # Validate that the folders in the nuget are expected - # @TODO: This does not verify we are not missing any folders, only that the folders that - # exist are expected to exist. - - powershell: | - # Propagate pipeline to PS variables ############################# - $expectedFolderNames = "${{ parameters.expectedFolderNames }}" - echo "expectedFolderNames= $expectedFolderNames" - - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify folders are expected #################################### - Get-ChildItem -Path $nugetPackageInstallPath -Directory | select Name | foreach { - if($expectedFolderNames.contains($_.Name)){ - Write-Host expected folder name verfied: $_.Name - } - } - displayName: "Verify NuGet Root Folder Structure" - - # Validate that the folders within the root folders of the nuget are expected - # @TODO: This does not verify we are not missing any folders, only that the folders that - # exist are expected to exist. - - powershell: | - # Propagate pipeline to PS variables ############################# - $expectedDotnetVersions = "${{ parameters.expectedDotnetVersions }}" - echo "expectedDotnetVersions= $expectedDotnetVersions" - - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify folders are expected #################################### - # Checks the version of DotNetFramework and DotNet - $countErr = 0 - $countPass = 0 - $excludNamesFromRuntimeFolder = 'lib','win','unix' - - Get-ChildItem -Path $nugetPackageInstallPath -Directory | foreach { - $parentname=$_.Name - Write-Host $_.FullName -ForegroundColor yellow - - if($_.Name -ne 'runtimes') { - Get-ChildItem -Path $_.FullName -Directory | select Name | foreach { - if($expectedDotnetVersions.Contains($_.Name)){ - Write-Host "`tExpected version verified in $parentname": $_.Name -ForegroundColor green - $countPass += 1 - } - else{ - Write-Host "`tUnexpected version detected in $parentname": $_.Name - $countErr += 1 - } - } - } - - elseif ($_.Name -eq 'runtimes'){ - Get-ChildItem -Depth 3 -Path $_.FullName -Exclude $excludNamesFromRuntimeFolder -Directory | foreach{ - if('${{ parameters.expectedDotnetVersions }}'.Contains($_.Name)){ - Write-Host "`tExpected version verfied in $parentname": $_.Name - $countPass += 1 - } - else{ - Write-Host "`tUnexpected version detected": $_.Name -ForegroundColor Red - $countErr += 1 - } - } - } - else{ - Write-Host "`tUnknown folder " $_.Name -ForegroundColor Red - Exit -1 - } - } - - Write-Host "_______________" - Write-Host "Expected: $countPass" - Write-Host "Unexpected: $countErr" - Write-Host "_______________" - if ($countErr -ne 0) - { - Write-Host "Unexpected versions are detected!" -ForegroundColor Red - Exit -1 - } - displayName: "Verify NuGet DotNet Versions " - - - powershell: | - # Propagate pipeline to PS variables ############################# - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify DLL Hierarchy ########################################### - foreach( $folderName in (Get-ChildItem -Path $nugetPackageInstallPath -Directory).Name) - { - # List all Childerns of the Path - Get-ChildItem -Path $nugetPackageInstallPath\$folderName -Recurse -File - $subFiles = Get-ChildItem -Path $nugetPackageInstallPath\$folderName -Recurse -File - - foreach($file in $subFiles) - { - if($subFiles[0].Name -like "*.dll" ) - { - Write-Host $subFiles[0].Name -ForegroundColor Green - Write-Host $subFiles[1].Name -ForegroundColor Green - if(($folderName -eq 'lib') -or ($folderName -eq 'ref')) - { - if($subFiles[2].Name -like "*.dll") - { - Write-Host $subFiles[2].Name -ForegroundColor Green - } - else - { - $subFiles[2].Name - Write-Host "Expected file pattern for localization did not match to *.dll" -ForegroundColor Red - Exit -1 - } - } - } - else - { - $subFiles[0].Name - $subFiles[1].Name - Write-Host "Expected file pattern did not match to *.dll" -ForegroundColor Red - Exit -1 - } - } - } - displayName: 'Verify all DLLs unzipped match "expected" hierarchy' - - # Verify that all DLLs are authenticode signed - # NOTE: This signing is only performed on official builds. - - ${{ if eq(parameters.isOfficial, true) }}: - - powershell: | - # Propagate pipeline to PS variables ############################# - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify authenticode signature of DLLs ########################## - $dlls = Get-ChildItem -Path $nugetPackageInstallPath -Recurse -Include *.dll - foreach ($status in $dlls | Get-AuthenticodeSignature) - { - if ($status.Status -eq "Valid") - { - Write-Host $status.Status $status.Path - } - else - { - Write-Host "dll status of '$status.Path' is not valid!" -ForegroundColor Red - $status - Exit -1 - } - } - displayName: "Verify all dlls status are Valid" - - - powershell: | - # Propagate pipeline to PS variables ############################# - $expectedFileVersion = "${{ parameters.expectedFileVersion }}" - echo "expectedFileVersion= $expectedFileVersion" - - $expectedPackageVersion = "${{ parameters.expectedPackageVersion }}" - echo "expectedPackageVersion= $expectedPackageVersion" - - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Validate ProductVersion and FileVersion fields ################# - $failed = 0 - foreach ( $pVersion in Get-ChildItem *.dll -Path $nugetPackageInstallPath -Recurse | ForEach-Object versioninfo ) - { - if ($pVersion.ProductVersion -Like $expectedPackageVersion + '*') - { - Write-Host -ForegroundColor Green "Correct ProductVersion detected for $($pVersion.FileName): $($pVersion.ProductVersion)" - } - else - { - Write-Host -ForegroundColor Red "Wrong ProductVersion detected for $($pVersion.FileName); expected: $expectedPackageVersion; found: $($pVersion.ProductVersion)" - $failed = 1 - } - - if ($pVersion.FileVersion -eq $expectedFileVersion) - { - Write-Host -ForegroundColor Green "Correct FileVersion detected for $($pVersion.FileName): $($pVersion.FileVersion)" - } - else - { - Write-Host -ForegroundColor Red "Wrong FileVersion detected for $($pVersion.FileName); expected $expectedFileVersion; found: $($pVersion.FileVersion)" - $failed = 1 - } - - # @TODO: We should do a check for assembly version here. - } - - if ($failed -ne 0) - { - Exit -1 - } - - Get-ChildItem *.dll -Path $nugetPackageInstallPath -Recurse | ForEach-Object VersionInfo | Format-List - displayName: 'Verify "File Version" matches expected values for DLLs' diff --git a/eng/pipelines/onebranch/scripts/tests/README.md b/eng/pipelines/onebranch/scripts/tests/README.md index 3585312d53..b2ea3754c1 100644 --- a/eng/pipelines/onebranch/scripts/tests/README.md +++ b/eng/pipelines/onebranch/scripts/tests/README.md @@ -37,10 +37,16 @@ Invoke-Pester ./publish-symbols.Tests.ps1 -Output Detailed | Request bodies | Registration body, default publish flags, flag overrides | | Error handling | Token failure, registration failure, publish failure, status failure — all verify expanded URI in error message | | Status validation | Detects Failed/Cancelled results, respects PublishToInternal/PublishToPublic flags, passes on Succeeded/Pending | +| Package validation | Wildcard vs per-id version expectations, SqlServer omitted when unbuilt, gate tokens, report written before gating, exit-code handling | +| Package signatures | Every package and symbol package verified, all failures reported before throwing | +| Assembly signatures | Package expansion, native binaries under `runtimes/` included, stale expansions replaced, all unsigned assemblies reported | ## Notes - All external calls (`az`, `Invoke-RestMethod`) are mocked — no network access or Azure credentials are required. - Script-level version tests mock `dotnet`; package-composition tests invoke the real MSBuild `GetVersionsSqlClient` and `GetVersionsSqlServer` targets. +- `Get-AuthenticodeSignature` is Windows-only, so the assembly-signature tests declare a stub when + it is absent. Only the signature lookup is substituted; package expansion and reporting run for + real against packages built in the test's temporary directory. - Tests validate scripts in the parent directory relative to this directory. diff --git a/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 new file mode 100644 index 0000000000..626dd2dca9 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 @@ -0,0 +1,188 @@ +<# +.SYNOPSIS + Pester tests for validate-packages.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'validate-packages.ps1' + + # Stands in for the built PackageValidator.dll; the script only checks that it exists. + $script:validatorPath = Join-Path $TestDrive 'PackageValidator.dll' + Set-Content -LiteralPath $script:validatorPath -Value 'stub' + + $script:packagesPath = Join-Path $TestDrive 'packages' + New-Item -ItemType Directory -Force -Path $script:packagesPath | Out-Null + Set-Content -LiteralPath (Join-Path $script:packagesPath 'Microsoft.Data.SqlClient.7.1.0.nupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'Microsoft.SqlServer.Server.1.1.0.nupkg') -Value 'stub' + + $script:reportPath = Join-Path $TestDrive 'out' 'report.json' + + function Invoke-ValidatePackages { + param( + [string]$PackagesPath = $script:packagesPath, + [string]$ValidatorPath = $script:validatorPath, + [string]$SqlClientPackageVersion = '7.1.0-preview3.26238.3', + [string]$SqlClientFileVersion = '7.1.0.26238', + [string]$SqlServerPackageVersion = '', + [string]$SqlServerFileVersion = '', + [string[]]$FailOn = @('error') + ) + + & $scriptPath ` + -ValidatorPath $ValidatorPath ` + -PackagesPath $PackagesPath ` + -ReportPath $script:reportPath ` + -SqlClientPackageVersion $SqlClientPackageVersion ` + -SqlClientFileVersion $SqlClientFileVersion ` + -SqlServerPackageVersion $SqlServerPackageVersion ` + -SqlServerFileVersion $SqlServerFileVersion ` + -FailOn $FailOn ` + -DotnetPath 'dotnet' *>&1 | Out-String + } + + # Captures the arguments of each invocation so tests can assert on what the validator was + # asked to do, and controls the exit code of each run. + function Set-DotnetMock { + param( + [int]$GateExitCode = 0, + [int]$ReportExitCode = 0 + ) + + $global:validatePackagesInvocations = @() + Mock -CommandName 'dotnet' -MockWith { + $global:validatePackagesInvocations += , @($args) + # The first run carries --json and never gates; the second applies the gate. + if ($args -contains '--json') { + $global:LASTEXITCODE = $ReportExitCode + return '{ "packages": [], "summary": {} }' + } + + $global:LASTEXITCODE = $GateExitCode + return 'validator output' + }.GetNewClosure() + } +} + +AfterAll { + Remove-Variable -Name 'validatePackagesInvocations' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'validate-packages.ps1 Expectations' { + BeforeEach { + Set-DotnetMock + } + + It 'applies the SqlClient family versions as wildcard expectations' { + Invoke-ValidatePackages | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $gateArgs | Should -Contain '*=7.1.0-preview3.26238.3' + $gateArgs | Should -Contain '*=7.1.0.26238' + } + + It 'omits SqlServer expectations when its versions are not supplied' { + Invoke-ValidatePackages | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + ($gateArgs -join ' ') | Should -Not -Match 'Microsoft\.SqlServer\.Server=' + } + + It 'adds SqlServer expectations as a per-id override when supplied' { + Invoke-ValidatePackages -SqlServerPackageVersion '1.1.0-preview1.26238.3' -SqlServerFileVersion '1.1.0.26238' | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $gateArgs | Should -Contain 'Microsoft.SqlServer.Server=1.1.0-preview1.26238.3' + $gateArgs | Should -Contain 'Microsoft.SqlServer.Server=1.1.0.26238' + } + + It 'passes each gate token as its own --fail-on argument' { + Invoke-ValidatePackages -FailOn @('error', 'missing-symbols', 'package-unsigned') | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $joined = $gateArgs -join ' ' + $joined | Should -Match '--fail-on error' + $joined | Should -Match '--fail-on missing-symbols' + $joined | Should -Match '--fail-on package-unsigned' + } + + It 'splits a single comma-separated gate token, as an Azure Pipelines argument line supplies it' { + Invoke-ValidatePackages -FailOn 'error, missing-symbols' | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $joined = $gateArgs -join ' ' + $joined | Should -Match '--fail-on error' + $joined | Should -Match '--fail-on missing-symbols' + $joined | Should -Not -Match 'error,' + } + + It 'writes the JSON report before applying the gate' { + Invoke-ValidatePackages | Out-Null + + # The reporting run must come first so the report survives a failing gate. + $firstInvocation = $global:validatePackagesInvocations | Select-Object -First 1 + $firstInvocation | Should -Contain '--json' + Test-Path -LiteralPath $script:reportPath | Should -BeTrue + } + + It 'does not gate the reporting run' { + Invoke-ValidatePackages -FailOn @('error') | Out-Null + + $reportArgs = $global:validatePackagesInvocations | Where-Object { $_ -contains '--json' } | Select-Object -First 1 + ($reportArgs -join ' ') | Should -Not -Match '--fail-on' + } +} + +Describe 'validate-packages.ps1 Exit Codes' { + It 'succeeds when the validator reports no gating findings' { + Set-DotnetMock -GateExitCode 0 + + $output = Invoke-ValidatePackages + $output | Should -Match 'Package validation passed' + } + + It 'fails when a gate is tripped' { + Set-DotnetMock -GateExitCode 2 + + { Invoke-ValidatePackages -FailOn @('error', 'missing-symbols') } | + Should -Throw '*matched the gate (error, missing-symbols)*' + } + + It 'reports an unexpected validator failure distinctly from a tripped gate' { + Set-DotnetMock -GateExitCode 1 + + { Invoke-ValidatePackages } | Should -Throw '*exited unexpectedly with code 1*' + } + + It 'fails the reporting run before gating so the real cause is not obscured' { + Set-DotnetMock -ReportExitCode 1 + + { Invoke-ValidatePackages } | Should -Throw '*failed while writing the JSON report (exit code 1)*' + + # The gating run must not have been reached. + $global:validatePackagesInvocations.Count | Should -Be 1 + } +} + +Describe 'validate-packages.ps1 Error Handling' { + BeforeEach { + Set-DotnetMock + } + + It 'throws when the validator is missing' { + { Invoke-ValidatePackages -ValidatorPath (Join-Path $TestDrive 'absent.dll') } | + Should -Throw '*PackageValidator was not found*' + } + + It 'throws when no packages are found' { + $empty = Join-Path $TestDrive 'empty' + New-Item -ItemType Directory -Force -Path $empty | Out-Null + + { Invoke-ValidatePackages -PackagesPath $empty } | Should -Throw '*No .nupkg files were found*' + } + + It 'rejects a half-supplied SqlServer expectation' { + # Supplying only one would assert a package version without its file version. + { Invoke-ValidatePackages -SqlServerPackageVersion '1.1.0' } | + Should -Throw '*must be supplied together*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 new file mode 100644 index 0000000000..4d4946a5fb --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 @@ -0,0 +1,149 @@ +<# +.SYNOPSIS + Pester tests for verify-assembly-signatures.ps1. + +.NOTES + Get-AuthenticodeSignature is a Windows-only cmdlet, so a stub is declared when it is absent. + This lets the tests run on any platform while still exercising the script's real expansion and + reporting logic; only the signature lookup itself is substituted. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'verify-assembly-signatures.ps1' + + if (-not (Get-Command 'Get-AuthenticodeSignature' -ErrorAction SilentlyContinue)) { + function Get-AuthenticodeSignature { + param([Parameter(Mandatory = $true)][string[]]$FilePath) + throw 'stub must be mocked' + } + } + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + # Builds a real .nupkg so the script's expansion path is genuinely exercised. + function New-TestPackage { + param( + [string]$Name, + [string[]]$AssemblyPaths = @('lib/net8.0/Test.dll'), + [string[]]$OtherPaths = @() + ) + + $staging = Join-Path $TestDrive "staging-$Name" + if (Test-Path -LiteralPath $staging) { Remove-Item -LiteralPath $staging -Recurse -Force } + New-Item -ItemType Directory -Force -Path $staging | Out-Null + + foreach ($relative in ($AssemblyPaths + $OtherPaths)) { + $full = Join-Path $staging $relative + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $full) | Out-Null + Set-Content -LiteralPath $full -Value 'stub' + } + + $packagePath = Join-Path $script:packagesPath "$Name.nupkg" + if (Test-Path -LiteralPath $packagePath) { Remove-Item -LiteralPath $packagePath -Force } + [System.IO.Compression.ZipFile]::CreateFromDirectory($staging, $packagePath) + return $packagePath + } + + function Invoke-VerifyAssemblySignatures { + param( + [string]$PackagesPath = $script:packagesPath, + [string]$ExtractPath = $script:extractPath + ) + + & $scriptPath -PackagesPath $PackagesPath -ExtractPath $ExtractPath *>&1 | Out-String + } +} + +AfterAll { + Remove-Variable -Name 'verifyAssemblySeen' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'verify-assembly-signatures.ps1' { + BeforeEach { + $script:packagesPath = Join-Path $TestDrive 'packages' + $script:extractPath = Join-Path $TestDrive 'extract' + foreach ($path in @($script:packagesPath, $script:extractPath)) { + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Recurse -Force } + New-Item -ItemType Directory -Force -Path $path | Out-Null + } + } + + It 'expands packages and verifies every assembly they contain' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') | Out-Null + New-TestPackage -Name 'PackageTwo' -AssemblyPaths @('lib/net8.0/Two.dll', 'runtimes/win-x64/native/sni.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 3 assemblies are Authenticode signed' + } + + It 'finds native binaries under runtimes, not just managed assemblies' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('runtimes/win-arm64/native/sni.dll') | Out-Null + $global:verifyAssemblySeen = @() + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $global:verifyAssemblySeen = $FilePath + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + Invoke-VerifyAssemblySignatures | Out-Null + ($global:verifyAssemblySeen -join ';') | Should -Match 'sni\.dll' + } + + It 'ignores non-assembly content' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') -OtherPaths @('README.md', 'lib/net8.0/One.xml') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 1 assemblies are Authenticode signed' + } + + It 'fails when an assembly is not validly signed' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Two.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $index = 0 + $FilePath | ForEach-Object { + $status = if ($index -eq 0) { 'Valid' } else { 'NotSigned' } + $index++ + [pscustomobject]@{ Path = $_; Status = $status } + } + } + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*failed for 1 of 2 assemblies*' + } + + It 'reports every unsigned assembly rather than stopping at the first' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Two.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'NotSigned' } } + } + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*failed for 2 of 2 assemblies*' + } + + It 'replaces a previous expansion so stale content cannot be verified' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Stale.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + Invoke-VerifyAssemblySignatures | Out-Null + + # Repack the same package id with fewer assemblies; the stale one must not linger. + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') | Out-Null + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 1 assemblies are Authenticode signed' + } + + It 'throws when no packages are found' { + { Invoke-VerifyAssemblySignatures } | Should -Throw '*No .nupkg files were found*' + } + + It 'throws when packages contain no assemblies' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @() -OtherPaths @('README.md') | Out-Null + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*No assemblies were found*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 new file mode 100644 index 0000000000..cadc1831a4 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 @@ -0,0 +1,87 @@ +<# +.SYNOPSIS + Pester tests for verify-package-signatures.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'verify-package-signatures.ps1' + + $script:packagesPath = Join-Path $TestDrive 'packages' + New-Item -ItemType Directory -Force -Path (Join-Path $script:packagesPath 'SqlClient') | Out-Null + New-Item -ItemType Directory -Force -Path (Join-Path $script:packagesPath 'SqlServer') | Out-Null + + # Symbol packages are signed too, so both extensions must be picked up, and the nested layout + # mirrors how each artifact is downloaded into its own subdirectory. + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlClient' 'Microsoft.Data.SqlClient.7.1.0.nupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlClient' 'Microsoft.Data.SqlClient.7.1.0.snupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlServer' 'Microsoft.SqlServer.Server.1.1.0.nupkg') -Value 'stub' + + function Invoke-VerifyPackageSignatures { + param([string]$PackagesPath = $script:packagesPath) + + & $scriptPath -PackagesPath $PackagesPath -DotnetPath 'dotnet' *>&1 | Out-String + } + + # Fails verification only for packages whose name matches, so tests can make a subset unsigned. + function Set-DotnetMock { + param([string]$FailPattern = '') + + $global:verifyPackageInvocations = @() + Mock -CommandName 'dotnet' -MockWith { + $global:verifyPackageInvocations += , @($args) + $target = $args[-1] + if ($FailPattern -and $target -match $FailPattern) { + $global:LASTEXITCODE = 1 + return "unsigned" + } + + $global:LASTEXITCODE = 0 + return "verified" + }.GetNewClosure() + } +} + +AfterAll { + Remove-Variable -Name 'verifyPackageInvocations' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'verify-package-signatures.ps1' { + It 'verifies every package and symbol package found' { + Set-DotnetMock + + $output = Invoke-VerifyPackageSignatures + $output | Should -Match 'All 3 package signature\(s\) verified' + $global:verifyPackageInvocations.Count | Should -Be 3 + } + + It 'invokes dotnet nuget verify with --all' { + Set-DotnetMock + + Invoke-VerifyPackageSignatures | Out-Null + + $first = $global:verifyPackageInvocations | Select-Object -First 1 + ($first -join ' ') | Should -Match 'nuget verify --all' + } + + It 'fails when a package signature does not verify' { + Set-DotnetMock -FailPattern 'SqlServer' + + { Invoke-VerifyPackageSignatures } | Should -Throw '*Microsoft.SqlServer.Server.1.1.0.nupkg*' + } + + It 'checks every package before failing so all failures are reported' { + Set-DotnetMock -FailPattern '\.nupkg$' + + # Two of the three files are .nupkg; both must appear rather than only the first. + { Invoke-VerifyPackageSignatures } | Should -Throw '*failed for 2 of 3 package(s)*' + $global:verifyPackageInvocations.Count | Should -Be 3 + } + + It 'throws when no packages are found' { + Set-DotnetMock + $empty = Join-Path $TestDrive 'empty' + New-Item -ItemType Directory -Force -Path $empty | Out-Null + + { Invoke-VerifyPackageSignatures -PackagesPath $empty } | Should -Throw '*No package files were found*' + } +} diff --git a/eng/pipelines/onebranch/scripts/validate-packages.ps1 b/eng/pipelines/onebranch/scripts/validate-packages.ps1 new file mode 100644 index 0000000000..af5149c9f5 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/validate-packages.ps1 @@ -0,0 +1,213 @@ +<# +.SYNOPSIS + Runs the PackageValidator tool over the NuGet packages produced by a OneBranch build. + +.DESCRIPTION + Invokes tools/PackageValidator once for a whole directory of packages rather than once per + package, because its most valuable checks are cross-package: every package in the SqlClient + family must carry the same version, and their inter-package dependency ranges must agree. + Validating one package at a time would silently skip all of those findings. + + The validator runs twice over the same inputs. The first run writes a machine-readable report + and never gates, so the report exists even when validation fails. The second run renders the + human-readable report and applies the gate, so a failing build shows its findings in its own + log rather than only in an artifact. + + Expected versions are supplied by the caller rather than derived here. The compute-versions + stage already computes every version the build stamps, and re-deriving them would reintroduce + the drift this validation exists to catch. + + Microsoft.SqlServer.Server is versioned separately from the SqlClient family, so its expected + versions are applied as a per-id override of the family wildcard. When it is not built in a + run, its package is absent from the drop and its expectations must be omitted entirely: the + validator rejects an expectation whose value is empty. + +.PARAMETER ValidatorPath + Path to the built PackageValidator.dll. Invoked through the managed assembly rather than the + native apphost so the same command works regardless of agent OS. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg files. Sibling .snupkg files must sit beside their + .nupkg for symbol matching to resolve, which is how the build jobs publish them. + +.PARAMETER ReportPath + Path of the JSON report to write. Parent directories are created as needed. + +.PARAMETER SqlClientPackageVersion + Package version expected of every package in the SqlClient family, applied as a wildcard. + Pointing every package at one value is what proves they agree, and also catches the case where + all of them are consistently wrong. + +.PARAMETER SqlClientFileVersion + Assembly file version expected of every assembly in the SqlClient family. + +.PARAMETER SqlServerPackageVersion + Package version expected of Microsoft.SqlServer.Server. Omit when SqlServer is not built. + +.PARAMETER SqlServerFileVersion + Assembly file version expected of Microsoft.SqlServer.Server. Omit when SqlServer is not built. + +.PARAMETER FailOn + Finding severities and/or categories that fail the build. Run the validator with --help to see + the available categories. Note that missing-symbols is a warning and package-unsigned is info, + so neither is covered by the error severity and both must be named explicitly. + + Accepts either an array or a single comma-separated string, because an Azure Pipelines task + argument line collapses to one token and PowerShell's -File mode does not split it. + +.PARAMETER DotnetPath + dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter + primarily supports isolated testing. + +.EXAMPLE + ./validate-packages.ps1 ` + -ValidatorPath ./PackageValidator.dll ` + -PackagesPath ./packages ` + -ReportPath ./out/report.json ` + -SqlClientPackageVersion 7.1.0-preview3.26238.3 ` + -SqlClientFileVersion 7.1.0.26238 ` + -FailOn error,missing-symbols + + Validates a family-only drop, failing on any error and on missing symbols. + +.NOTES + File Name : validate-packages.ps1 + Requires : PowerShell 7+ and the repository-pinned .NET SDK. + Called by : validate-packages-step.yml + + PackageValidator exit codes: + 0 - No gating findings. + 1 - The validator itself failed. + 2 - A --fail-on gate was tripped. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Path to the built PackageValidator.dll.")] + [ValidateNotNullOrEmpty()] + [string]$ValidatorPath, + + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for .nupkg files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(Mandatory = $true, HelpMessage = "Path of the JSON report to write.")] + [ValidateNotNullOrEmpty()] + [string]$ReportPath, + + [Parameter(Mandatory = $true, HelpMessage = "Package version expected of the SqlClient family.")] + [ValidateNotNullOrEmpty()] + [string]$SqlClientPackageVersion, + + [Parameter(Mandatory = $true, HelpMessage = "File version expected of the SqlClient family.")] + [ValidateNotNullOrEmpty()] + [string]$SqlClientFileVersion, + + [Parameter(HelpMessage = "Package version expected of Microsoft.SqlServer.Server, when built.")] + [string]$SqlServerPackageVersion = "", + + [Parameter(HelpMessage = "File version expected of Microsoft.SqlServer.Server, when built.")] + [string]$SqlServerFileVersion = "", + + [Parameter(Mandatory = $true, HelpMessage = "Severities and/or categories that fail the build.")] + [ValidateNotNullOrEmpty()] + [string[]]$FailOn, + + [Parameter(HelpMessage = "dotnet executable to invoke.")] + [ValidateNotNullOrEmpty()] + [string]$DotnetPath = "dotnet" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Split on commas so a single "error,missing-symbols" token behaves like a two-element array. +$failOnTokens = @($FailOn -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) +if ($failOnTokens.Count -eq 0) { + throw "FailOn must name at least one severity or category." +} + +Write-Host "=== Validate Packages Parameters ===" +Write-Host "ValidatorPath: ${ValidatorPath}" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "ReportPath: ${ReportPath}" +Write-Host "SqlClientPackageVersion: ${SqlClientPackageVersion}" +Write-Host "SqlClientFileVersion: ${SqlClientFileVersion}" +Write-Host "SqlServerPackageVersion: ${SqlServerPackageVersion}" +Write-Host "SqlServerFileVersion: ${SqlServerFileVersion}" +Write-Host "FailOn: $($failOnTokens -join ', ')" +Write-Host "====================================" + +if (-not (Test-Path -LiteralPath $ValidatorPath)) { + throw "PackageValidator was not found at '${ValidatorPath}'." +} + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Filter *.nupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No .nupkg files were found under '${PackagesPath}'." +} + +Write-Host "Validating $($packages.Count) package(s):" +$packages | ForEach-Object { Write-Host " $($_.Name)" } + +# A bare value applies to every package; an id=value pair overrides it for that package only. +$expectations = @( + "--expect-package-version", "*=${SqlClientPackageVersion}" + "--expect-file-version", "*=${SqlClientFileVersion}" +) + +# Both SqlServer versions travel together: supplying only one would assert half a package. +$hasSqlServerPackageVersion = -not [string]::IsNullOrWhiteSpace($SqlServerPackageVersion) +$hasSqlServerFileVersion = -not [string]::IsNullOrWhiteSpace($SqlServerFileVersion) +if ($hasSqlServerPackageVersion -ne $hasSqlServerFileVersion) { + throw "SqlServerPackageVersion and SqlServerFileVersion must be supplied together, or not at all." +} + +if ($hasSqlServerPackageVersion) { + $expectations += @( + "--expect-package-version", "Microsoft.SqlServer.Server=${SqlServerPackageVersion}" + "--expect-file-version", "Microsoft.SqlServer.Server=${SqlServerFileVersion}" + ) +} + +$gate = @() +foreach ($token in $failOnTokens) { + $gate += @("--fail-on", $token) +} + +Write-Host "Expectations: $($expectations -join ' ')" +Write-Host "Gate: $($gate -join ' ')" + +$reportDirectory = Split-Path -Parent $ReportPath +if ($reportDirectory) { + New-Item -ItemType Directory -Force -Path $reportDirectory | Out-Null +} + +# Reported before gating so the JSON exists even for a failing run. +& $DotnetPath $ValidatorPath $PackagesPath --json @expectations | + Set-Content -LiteralPath $ReportPath -Encoding utf8 +$reportExitCode = $LASTEXITCODE + +# This run is ungated, so any non-zero code means the validator itself failed and the report it +# produced cannot be trusted. Fail here rather than let the gated run obscure the real cause. +if ($reportExitCode -ne 0) { + throw "PackageValidator failed while writing the JSON report (exit code ${reportExitCode})." +} + +Write-Host "Wrote JSON report to ${ReportPath}" + +Write-Host "" +Write-Host "=== Package validation report ===" +& $DotnetPath $ValidatorPath $PackagesPath @expectations @gate +$exitCode = $LASTEXITCODE + +if ($exitCode -eq 0) { + Write-Host "" + Write-Host "Package validation passed." +} +elseif ($exitCode -eq 2) { + throw "Package validation failed: one or more findings matched the gate ($($failOnTokens -join ', '))." +} +else { + throw "PackageValidator exited unexpectedly with code ${exitCode}." +} diff --git a/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 b/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 new file mode 100644 index 0000000000..76c1495602 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 @@ -0,0 +1,99 @@ +<# +.SYNOPSIS + Verifies that every assembly shipped inside an official build's NuGet packages is Authenticode + signed. + +.DESCRIPTION + Expands each .nupkg beneath a directory and checks the Authenticode signature of every assembly + it contains, including native binaries under runtimes/. + + Packages are expanded rather than installed through NuGet so that every produced package is + covered without resolving dependencies, and so that the check does not depend on any single + package id. + + This complements PackageValidator, which reports strong-name state from assembly metadata + cross-platform. Authenticode verification requires the Windows trust store, so this script runs + only on Windows agents and only for official builds; non-official builds deliberately produce + unsigned assemblies. + + Every assembly is checked before failing, so a single run reports all unsigned assemblies + rather than stopping at the first. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg files to expand. + +.PARAMETER ExtractPath + Directory the packages are expanded into. Each package is expanded into its own subdirectory so + that identically-named assemblies from different packages cannot collide. Existing content for + a package is replaced. + +.EXAMPLE + ./verify-assembly-signatures.ps1 -PackagesPath ./packages -ExtractPath ./extract + + Expands every package beneath ./packages and verifies the signature of each assembly. + +.NOTES + File Name : verify-assembly-signatures.ps1 + Requires : PowerShell 7+ on Windows (Get-AuthenticodeSignature). + Called by : validate-packages-job.yml +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for .nupkg files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(Mandatory = $true, HelpMessage = "Directory the packages are expanded into.")] + [ValidateNotNullOrEmpty()] + [string]$ExtractPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +Write-Host "=== Verify Assembly Signatures Parameters ===" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "ExtractPath: ${ExtractPath}" +Write-Host "=============================================" + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Filter *.nupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No .nupkg files were found under '${PackagesPath}'." +} + +New-Item -ItemType Directory -Force -Path $ExtractPath | Out-Null + +Add-Type -AssemblyName System.IO.Compression.FileSystem +foreach ($package in $packages) { + $destination = Join-Path $ExtractPath $package.BaseName + if (Test-Path -LiteralPath $destination) { + Remove-Item -LiteralPath $destination -Recurse -Force + } + + Write-Host "Expanding $($package.Name)" + [System.IO.Compression.ZipFile]::ExtractToDirectory($package.FullName, $destination) +} + +$assemblies = @(Get-ChildItem -Path $ExtractPath -Recurse -File -Filter *.dll) +if ($assemblies.Count -eq 0) { + throw "No assemblies were found under '${ExtractPath}'." +} + +# Every assembly is checked before throwing so one run reports all failures. +$unsigned = @() +foreach ($signature in @(Get-AuthenticodeSignature -FilePath $assemblies.FullName)) { + if ($signature.Status -eq "Valid") { + Write-Host " OK $($signature.Path)" + } + else { + Write-Host " FAIL $($signature.Path) - $($signature.Status)" + $unsigned += $signature.Path + } +} + +if ($unsigned.Count -gt 0) { + throw "Authenticode verification failed for $($unsigned.Count) of $($assemblies.Count) assemblies." +} + +Write-Host "All $($assemblies.Count) assemblies are Authenticode signed." diff --git a/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 b/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 new file mode 100644 index 0000000000..74fa864c07 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 @@ -0,0 +1,72 @@ +<# +.SYNOPSIS + Verifies the NuGet signatures of every package produced by an official OneBranch build. + +.DESCRIPTION + Runs `dotnet nuget verify --all` over every .nupkg and .snupkg found beneath a directory, + confirming that each carries a valid, trusted signature. + + This complements PackageValidator, which reports signature *presence* from package metadata + cross-platform. Establishing that a signature is trusted requires the platform trust store, + which is why this runs separately and only on official builds. Non-official builds deliberately + produce unsigned packages, so verifying them would always fail. + + Every package is verified before failing, so a single run reports all unsigned packages rather + than stopping at the first. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg and .snupkg files. + +.PARAMETER DotnetPath + dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter + primarily supports isolated testing. + +.EXAMPLE + ./verify-package-signatures.ps1 -PackagesPath ./packages + + Verifies every package and symbol package beneath ./packages. + +.NOTES + File Name : verify-package-signatures.ps1 + Requires : PowerShell 7+ and the repository-pinned .NET SDK. + Called by : validate-packages-job.yml +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for package files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(HelpMessage = "dotnet executable to invoke.")] + [ValidateNotNullOrEmpty()] + [string]$DotnetPath = "dotnet" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +Write-Host "=== Verify Package Signatures Parameters ===" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "============================================" + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Include *.nupkg, *.snupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No package files were found under '${PackagesPath}'." +} + +# Every package is checked before throwing so one run reports all failures. +$failed = @() +foreach ($package in $packages) { + Write-Host "Verifying $($package.Name)" + & $DotnetPath nuget verify --all $package.FullName + if ($LASTEXITCODE -ne 0) { + $failed += $package.Name + } +} + +if ($failed.Count -gt 0) { + throw "NuGet signature verification failed for $($failed.Count) of $($packages.Count) package(s): $($failed -join ', ')" +} + +Write-Host "All $($packages.Count) package signature(s) verified." diff --git a/eng/pipelines/onebranch/stages/build-stages.yml b/eng/pipelines/onebranch/stages/build-stages.yml index afd1907859..34f88bd9b3 100644 --- a/eng/pipelines/onebranch/stages/build-stages.yml +++ b/eng/pipelines/onebranch/stages/build-stages.yml @@ -331,17 +331,44 @@ stages: packageVersion: '$(sqlClientPackageVersion)' # ==================================================================== - # Validation - # @TODO: Update validate-signed-package-job to compute expected versions from - # Versions.props (same as build jobs) instead of receiving them as parameters. + # Stage 5: Validation + # Validates every package produced by this run, together, so that + # cross-package rules (shared family version, dependency agreement) + # are actually exercised. Depends on all build stages. # ==================================================================== - # - stage: sqlclient_package_validation - # displayName: "SqlClient Package Validation" - # dependsOn: build_dependent - # jobs: - # - template: /eng/pipelines/onebranch/jobs/validate-signed-package-job.yml@self - # parameters: - # artifactName: '${{ parameters.sqlClientArtifactsName }}' - # expectedFileVersion: - # expectedPackageVersion: - # isOfficial: ${{ parameters.isOfficial }} + - stage: package_validation + displayName: "Validate Packages" + dependsOn: + - compute_versions + - build_independent + - build_abstractions + - build_dependent + - build_addons + + variables: + - name: sqlClientPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlServerPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlServerFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerFileVersion'] ] + + jobs: + - template: /eng/pipelines/onebranch/jobs/validate-packages-job.yml@self + parameters: + abstractionsArtifactsName: '${{ parameters.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ parameters.akvProviderArtifactsName }}' + azureArtifactsName: '${{ parameters.azureArtifactsName }}' + loggingArtifactsName: '${{ parameters.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ parameters.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ parameters.sqlServerArtifactsName }}' + + sqlClientPackageVersion: '$(sqlClientPackageVersion)' + sqlClientFileVersion: '$(sqlClientFileVersion)' + sqlServerPackageVersion: '$(sqlServerPackageVersion)' + sqlServerFileVersion: '$(sqlServerFileVersion)' + + buildSqlServer: ${{ parameters.buildSqlServer }} + isOfficial: ${{ parameters.isOfficial }} diff --git a/eng/pipelines/onebranch/stages/release-stages.yml b/eng/pipelines/onebranch/stages/release-stages.yml index d160c8906f..7ee8023cd9 100644 --- a/eng/pipelines/onebranch/stages/release-stages.yml +++ b/eng/pipelines/onebranch/stages/release-stages.yml @@ -112,6 +112,10 @@ stages: ${{ else }}: displayName: Release to NuGet Test dependsOn: + # Nothing is published unless every produced package passed validation. This stage also + # depends on all four build stages, but we keep the complete list here anyway to prevent + # regressions if package validation changes its dependencies. + - package_validation - ${{ if or(parameters.releaseSqlServer, parameters.releaseSqlClient) }}: - build_independent - ${{ if parameters.releaseSqlClient }}: diff --git a/eng/pipelines/onebranch/steps/validate-packages-step.yml b/eng/pipelines/onebranch/steps/validate-packages-step.yml new file mode 100644 index 0000000000..173df4d7ed --- /dev/null +++ b/eng/pipelines/onebranch/steps/validate-packages-step.yml @@ -0,0 +1,80 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Builds and runs tools/PackageValidator over a directory of produced NuGet packages. +# +# The validator is invoked once for the whole directory rather than once per package, because its +# most valuable checks are cross-package: it confirms that every package in the SqlClient family +# carries the same version and that their inter-package dependency ranges agree. Running it per +# package would silently skip all of those findings. +# +# Two invocations are made over the same inputs. The first writes a machine-readable report and is +# ungated, so the artifact exists even for a failing run. It still fails the step if the validator +# itself errors, because the report it produced could not then be trusted. The second renders the +# human-readable report and applies the gate, so a failed build shows the findings in its own log. + +parameters: + # Directory scanned recursively for .nupkg files. Sibling .snupkg files must sit beside their + # .nupkg for symbol matching to resolve, which is how the build jobs publish them. + - name: packagesPath + type: string + + # Path of the JSON report to write. + - name: reportPath + type: string + + # Expected versions shared by the whole SqlClient family (Logging, Abstractions, SqlClient, + # Azure, AkvProvider), applied as wildcard expectations. Pointing every package at the same + # value is what proves they agree, and also catches every package being consistently wrong. + - name: sqlClientPackageVersion + type: string + + - name: sqlClientFileVersion + type: string + + # Expected versions for the separately-versioned Microsoft.SqlServer.Server, applied as a per-id + # override of the family wildcard. Left empty when SqlServer is not built this run: its package + # is then absent from the drop, and the validator rejects an expectation with an empty value. + - name: sqlServerPackageVersion + type: string + default: '' + + - name: sqlServerFileVersion + type: string + default: '' + + # Finding severities and/or categories that fail the build. Run the validator with --help to + # see the full set of categories. + - name: failOn + type: object + default: + - error + +steps: + - task: DotNetCoreCLI@2 + displayName: 'build.proj - BuildPackageValidator' + inputs: + command: build + projects: '$(REPO_ROOT)/build.proj' + arguments: >- + -t:BuildPackageValidator + -p:Configuration=Release + + - task: PowerShell@2 + displayName: 'Validate NuGet packages' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/validate-packages.ps1 + arguments: >- + -ValidatorPath "$(REPO_ROOT)/tools/PackageValidator/src/bin/Release/net10.0/PackageValidator.dll" + -PackagesPath "${{ parameters.packagesPath }}" + -ReportPath "${{ parameters.reportPath }}" + -SqlClientPackageVersion "${{ parameters.sqlClientPackageVersion }}" + -SqlClientFileVersion "${{ parameters.sqlClientFileVersion }}" + -SqlServerPackageVersion "${{ parameters.sqlServerPackageVersion }}" + -SqlServerFileVersion "${{ parameters.sqlServerFileVersion }}" + -FailOn "${{ join(',', parameters.failOn) }}" From 872566322e2d38b7c8e75e68af7b6944754375e6 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:53:19 -0300 Subject: [PATCH 33/51] Pipelines | Name the agent image parameter poolImage everywhere and require it (#4626) * Name the agent image parameter poolImage everywhere and require it The image parameter was spelled five different ways across the pipeline templates - vmImage, image, agentImage, imageOverride and platformImage - so a reader had to check each template to learn what to pass. Rename all twenty declarations, references and call-site keys to 'poolImage', alongside the 'poolName' parameter it accompanies. Remove the image defaults from the nineteen templates that have callers, and pass the value explicitly at the nine call sites that had been inheriting one. A default meant a template silently picked an agent image that its caller never mentioned, which is easy to miss when auditing which images a pool must provide. Every call site now names its image. The queue-time parameter on the CI package pipeline keeps its default, since it has no caller to supply one, and the default lets a manual run pick a Windows agent when that is what we want to validate. This does not change which image any job runs on. All 38 call sites resolve to the same image as before, whether they previously passed a value or inherited a default. Note that ADO's own 'vmImage' pool keyword and the 'imageOverride' demand capability keep their names; only our parameters are renamed. * Fixed botched rebase conflict resolution. * Pipelines | Finish the vmImage -> poolImage rename in kerberos and MI templates Complete the rename started in PR #4626 so template expansion matches the renamed declarations: - sqlclient-ci-kerberos-job.yml: rename the vmImage parameter to poolImage and update the ImageOverride demand. - sqlclient-ci-kerberos-stages.yml: rename all four call-site keys. - sqlclient-ci-managed-instance-stages.yml: rename all four call-site keys to match the already-renamed poolImage parameter. Addresses review feedback on PR #4626. --- .../ci/kerberos/sqlclient-ci-kerberos-job.yml | 13 +++++++++---- .../kerberos/sqlclient-ci-kerberos-stages.yml | 8 ++++---- .../sqlclient-ci-managed-instance-job.yml | 4 ++-- .../sqlclient-ci-managed-instance-stages.yml | 8 ++++---- .../package/sqlclient-ci-package-pipeline.yml | 9 ++++++--- .../ci/stress/sqlclient-ci-stress-job.yml | 6 +++--- .../ci/stress/sqlclient-ci-stress-pipeline.yml | 2 +- .../ci/stress/sqlclient-ci-stress-stage.yml | 6 +++--- .../templates/jobs/ci-build-nugets-job.yml | 7 +++---- .../templates/jobs/ci-code-coverage-job.yml | 5 ++--- .../common/templates/jobs/ci-run-tests-job.yml | 8 ++++---- .../templates/stages/ci-run-tests-stage.yml | 4 ++-- eng/pipelines/dotnet-sqlclient-ci-core.yml | 4 ++++ .../jobs/pack-abstractions-package-ci-job.yml | 5 ++--- .../jobs/pack-azure-package-ci-job.yml | 5 ++--- .../jobs/pack-logging-package-ci-job.yml | 5 ++--- .../jobs/pack-sqlserver-package-ci-job.yml | 5 ++--- .../jobs/test-abstractions-package-ci-job.yml | 6 +++--- .../jobs/test-azure-package-ci-job.yml | 8 ++++---- eng/pipelines/pr/jobs/test-buildproj-job.yml | 4 ++-- .../pr/jobs/test-sqlclientmanual-job.yml | 4 ++-- eng/pipelines/pr/sqlclient-pr-pipeline.yml | 6 +++--- .../pr/stages/collect-coverage-stage.yml | 4 ++-- .../pr/stages/generate-secrets-stage.yml | 4 ++-- eng/pipelines/pr/stages/pack-stage.yml | 8 ++------ eng/pipelines/pr/stages/test-stages.yml | 18 +++++++++--------- .../build-abstractions-package-ci-stage.yml | 7 ++++--- .../stages/build-azure-package-ci-stage.yml | 7 ++++--- .../stages/build-logging-package-ci-stage.yml | 1 + .../build-sqlclient-package-ci-stage.yml | 1 + .../build-sqlserver-package-ci-stage.yml | 1 + .../stages/compute-versions-ci-stage.yml | 5 ++--- .../stages/generate-secrets-ci-stage.yml | 5 ++--- .../stages/verify-nuget-packages-ci-stage.yml | 5 ++--- 34 files changed, 101 insertions(+), 97 deletions(-) diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml index f6a1b680e7..0a8efa104c 100644 --- a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml @@ -38,6 +38,14 @@ parameters: - Linux - Windows + # The pool VM image to use. The pool named by poolName must provide an image with this name. + # + # NOTE: This value is evaluated at template-expansion (compile) time to select the pool image, so + # it must be a literal and must not contain any runtime expressions (e.g. $(...) macros or + # $[...] runtime expressions). + - name: poolImage + type: string + - name: poolName type: string @@ -48,9 +56,6 @@ parameters: type: boolean default: false - - name: vmImage - type: string - jobs: - job: kerberos_tests_job_${{ parameters.jobNameSuffix }} displayName: ${{ parameters.displayName }} @@ -62,7 +67,7 @@ jobs: pool: name: ${{ parameters.poolName }} demands: - - ImageOverride -equals ${{ parameters.vmImage }} + - ImageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml index 86c1b95cc9..90269805c9 100644 --- a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml @@ -52,10 +52,10 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} operatingSystem: Windows + poolImage: ADO-Win25 poolName: ADO-Trusted-Domain-Win-WestUS2 runtime: ${{ runtime }} useManagedSNI: false - vmImage: ADO-Win25 # .NET runs with both native and managed SNI. - ${{ each runtime in parameters.netTestRuntimes }}: @@ -67,10 +67,10 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} operatingSystem: Windows + poolImage: ADO-Win25 poolName: ADO-Trusted-Domain-Win-WestUS2 runtime: ${{ runtime }} useManagedSNI: false - vmImage: ADO-Win25 - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self parameters: @@ -80,10 +80,10 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} operatingSystem: Windows + poolImage: ADO-Win25 poolName: ADO-Trusted-Domain-Win-WestUS2 runtime: ${{ runtime }} useManagedSNI: true - vmImage: ADO-Win25 - stage: linux displayName: Linux @@ -102,6 +102,6 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} operatingSystem: Linux + poolImage: ADO-UB24 poolName: ADO-Trusted-Linux-WestUS2 runtime: ${{ runtime }} - vmImage: ADO-UB24 diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml index 0141650535..3d74f55e0d 100644 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml @@ -72,7 +72,7 @@ parameters: # NOTE: This value is evaluated at template-expansion (compile) time to select the pool image, so # it must be a literal and must not contain any runtime expressions (e.g. $(...) macros or # $[...] runtime expressions). - - name: vmImage + - name: poolImage type: string jobs: @@ -93,7 +93,7 @@ jobs: pool: name: Managed-Instance-pool demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml index e96aef1bc3..2176ff7c87 100644 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml @@ -51,9 +51,9 @@ stages: displayName: 'Win : Native SNI : ${{ runtime }}' jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} operatingSystem: Windows + poolImage: ADO-Win25 runtime: ${{ runtime }} useManagedSNI: false - vmImage: ADO-Win25 # .NET runs with both native and managed SNI. - ${{ each runtime in parameters.netTestRuntimes }}: @@ -65,9 +65,9 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} operatingSystem: Windows + poolImage: ADO-Win25 runtime: ${{ runtime }} useManagedSNI: false - vmImage: ADO-Win25 - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self parameters: @@ -77,9 +77,9 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} operatingSystem: Windows + poolImage: ADO-Win25 runtime: ${{ runtime }} useManagedSNI: true - vmImage: ADO-Win25 - stage: linux displayName: Linux @@ -96,5 +96,5 @@ stages: jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} dotnetVerbosity: ${{ parameters.dotnetVerbosity }} operatingSystem: Linux + poolImage: ADO-UB24 runtime: ${{ runtime }} - vmImage: ADO-UB24 diff --git a/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml b/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml index ada32431b2..8526379bad 100644 --- a/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml +++ b/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml @@ -40,8 +40,11 @@ parameters: # The agent image to use for the build. This must exist in both the ADO-1ES-Pool and # ADO-CI-1ES-Pool agent pools. - - name: agentImage - displayName: Agent Image + # + # This keeps a default so that manual runs can pick a Windows agent when that is what we + # want to validate. + - name: poolImage + displayName: Pool Image type: string default: ADO-UB24 values: @@ -98,7 +101,7 @@ jobs: ${{ else }}: name: ADO-CI-1ES-Pool demands: - - ImageOverride -equals ${{ parameters.agentImage }} + - ImageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml index 5b46f6c3fa..d7a064523e 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml @@ -82,7 +82,7 @@ parameters: type: string # The pool VM image to use, which must exist in the specified pool. - - name: vmImage + - name: poolImage type: string jobs: @@ -167,11 +167,11 @@ jobs: # Images provided by Azure Pipelines must be selected using 'vmImage'. ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} + vmImage: ${{ parameters.poolImage }} # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml index 57d52d8e4a..1e91534728 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml @@ -100,7 +100,7 @@ stages: parameters: debug: ${{ parameters.debug }} poolName: $(ci_var_defaultPoolName) - vmImage: ADO-UB24 + poolImage: ADO-UB24 # Run the stress tests. - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml@self diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml index 0a1916594d..6f8ce4aa6d 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml @@ -96,7 +96,7 @@ stages: parameters: saPassword: $(saPassword) poolName: ${{ parameters.poolName }} - vmImage: ADO-UB24-SQL25 + poolImage: ADO-UB24-SQL25 # ---------------------------------------------------------------------------------------------- # Build and test on Windows @@ -120,7 +120,7 @@ stages: # The Windows images include a suitable .NET Framework runtime, so we don't have to install # one explicitly. poolName: ${{ parameters.poolName }} - vmImage: ADO-MMS25-SQL25 + poolImage: ADO-MMS25-SQL25 # ---------------------------------------------------------------------------------------------- # Build and test on macOS. @@ -144,4 +144,4 @@ stages: # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted # 'Azure Pipelines' pool. poolName: Azure Pipelines - vmImage: macos-latest + poolImage: macos-latest diff --git a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml index 5840663f5c..ce8c58b743 100644 --- a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml @@ -27,10 +27,9 @@ parameters: - name: poolName type: string - # The imageOverride capability required from the 1ES pool. - - name: imageOverride + # The name of the VM image to run on, within the pool. + - name: poolImage type: string - default: ADO-Win25 # The name of the Abstractions pipeline artifact to download when referenceType is 'Package'. - name: abstractionsArtifactsName @@ -86,7 +85,7 @@ jobs: pool: name: ${{parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.imageOverride }} + - imageOverride -equals ${{ parameters.poolImage }} variables: - template: /eng/pipelines/libraries/ci-build-variables.yml@self diff --git a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml index 63c74a8013..0b71bbaa5e 100644 --- a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml @@ -28,9 +28,8 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string - default: ADO-UB24 jobs: - job: publish_code_coverage @@ -40,7 +39,7 @@ jobs: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: # Use a temp directory that is cleaned up after each job runs. This helps diff --git a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml index ba028f1502..86ee3806fd 100644 --- a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml @@ -57,7 +57,7 @@ parameters: default: false # The VM image to use, which must exist in the specified pool. - - name: image + - name: poolImage type: string # The display name for this job. @@ -147,7 +147,7 @@ parameters: type: string jobs: -- job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.image, 'unknown_image')) }} +- job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.poolImage, 'unknown_image')) }} # Some of our tests take longer than the default 60 minutes to run on some # OSes and configurations. @@ -158,11 +158,11 @@ jobs: # Images provided by Azure Pipelines must be selected using 'vmImage'. ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.image }} + vmImage: ${{ parameters.poolImage }} # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.image }} + - imageOverride -equals ${{ parameters.poolImage }} variables: - name: dotnetx86RootPath diff --git a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml index 1cda6c7580..29e0042569 100644 --- a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml +++ b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml @@ -106,7 +106,7 @@ stages: referenceType: ${{ parameters.referenceType }} timeout: ${{ parameters.testJobTimeout }} poolName: ${{ config.value.pool }} - image: ${{ image.value }} + poolImage: ${{ image.value }} jobDisplayName: ${{ format('{0}_{1}_{2}', replace(targetFramework, '.', '_'), platform, testSet) }} configProperties: ${{ config.value.configProperties }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} @@ -136,7 +136,7 @@ stages: referenceType: ${{ parameters.referenceType }} timeout: ${{ parameters.testJobTimeout }} poolName: ${{ config.value.pool }} - image: ${{ image.value }} + poolImage: ${{ image.value }} ${{if eq(usemanagedSNI, 'true') }}: jobDisplayName: ${{ format('{0}_{1}_{2}_{3}', replace(targetFramework, '.', '_'), platform, 'ManagedSNI', testSet) }} ${{ else }}: diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index 9b53b6a85a..3102b9f8a2 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -174,12 +174,14 @@ stages: - template: /eng/pipelines/stages/compute-versions-ci-stage.yml@self parameters: poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 buildSuffix: ${{ parameters.buildSuffix }} # Generate secrets used throughout the pipeline. - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self parameters: poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 debug: ${{ parameters.debug }} # Build the SqlServer package, and publish it to the pipeline artifacts @@ -276,6 +278,7 @@ stages: - template: /eng/pipelines/stages/verify-nuget-packages-ci-stage.yml@self parameters: poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-Win25 abstractionsArtifactsName: $(abstractionsArtifactsName) azureArtifactsName: $(azureArtifactsName) loggingArtifactsName: $(loggingArtifactsName) @@ -324,6 +327,7 @@ stages: parameters: debug: ${{ parameters.debug }} poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 # We only want to upload coverage results to CodeCov from certain # pipelines. We use the pipeline name (Build.DefinitionName) to # choose. This is a predefined variable that is available at diff --git a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml index bbd80cc3f9..3b01faddc5 100644 --- a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml @@ -76,9 +76,8 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string - default: ADO-UB24 jobs: @@ -91,7 +90,7 @@ jobs: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/pack-azure-package-ci-job.yml b/eng/pipelines/jobs/pack-azure-package-ci-job.yml index b626dd14a5..884d4a5c32 100644 --- a/eng/pipelines/jobs/pack-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-azure-package-ci-job.yml @@ -82,9 +82,8 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string - default: ADO-UB24 jobs: @@ -97,7 +96,7 @@ jobs: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/pack-logging-package-ci-job.yml b/eng/pipelines/jobs/pack-logging-package-ci-job.yml index 03fde36308..1adcf3ac1d 100644 --- a/eng/pipelines/jobs/pack-logging-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-logging-package-ci-job.yml @@ -59,9 +59,8 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string - default: ADO-UB24 jobs: @@ -74,7 +73,7 @@ jobs: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml b/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml index c869026812..54938389a5 100644 --- a/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml @@ -58,9 +58,8 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string - default: ADO-UB24 jobs: @@ -73,7 +72,7 @@ jobs: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml index 1cdf04c0be..c706ca0c57 100644 --- a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml @@ -66,7 +66,7 @@ parameters: type: string # The pool VM image to use. - - name: vmImage + - name: poolImage type: string jobs: @@ -78,11 +78,11 @@ jobs: # Images provided by Azure Pipelines must be selected using 'vmImage'. ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} + vmImage: ${{ parameters.poolImage }} # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/test-azure-package-ci-job.yml b/eng/pipelines/jobs/test-azure-package-ci-job.yml index 5319fba0c3..e927eb6e6e 100644 --- a/eng/pipelines/jobs/test-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/test-azure-package-ci-job.yml @@ -126,7 +126,7 @@ parameters: default: [] # The pool VM image to use. - - name: vmImage + - name: poolImage type: string jobs: @@ -138,11 +138,11 @@ jobs: # Images provided by Azure Pipelines must be selected using 'vmImage'. ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} + vmImage: ${{ parameters.poolImage }} # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: @@ -250,7 +250,7 @@ jobs: AADServicePrincipalId: $(AADServicePrincipalId) AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) # macOS doesn't support managed identities. - ManagedIdentitySupported: ${{ not(eq(parameters.vmImage, 'macos-latest')) }} + ManagedIdentitySupported: ${{ not(eq(parameters.poolImage, 'macos-latest')) }} TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING) UserManagedIdentityClientId: $(UserManagedIdentityClientId) WorkloadIdentityFederationServiceConnectionId: $(WorkloadIdentityFederationServiceConnectionId) diff --git a/eng/pipelines/pr/jobs/test-buildproj-job.yml b/eng/pipelines/pr/jobs/test-buildproj-job.yml index c232513e1f..1e9a93fdab 100644 --- a/eng/pipelines/pr/jobs/test-buildproj-job.yml +++ b/eng/pipelines/pr/jobs/test-buildproj-job.yml @@ -45,7 +45,7 @@ parameters: type: string # Name of the image in the customized ADO pool to use to execute this job. - - name: platformImage + - name: poolImage type: string # Name of the pool to use for jobs that require customized VM images. @@ -89,7 +89,7 @@ jobs: pool: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.platformImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: # Install dotnet and the runtime that will run the tests (if it is not netframework) diff --git a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml index 38b49695fe..3b975aa46b 100644 --- a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml +++ b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml @@ -49,7 +49,7 @@ parameters: type: string # Name of the image in the customized ADO pool to use to execute this job. - - name: platformImage + - name: poolImage type: string # General name of the operating system that will be used to execute the job. @@ -132,7 +132,7 @@ jobs: pool: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.platformImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: # Bring the generated SA password from the secrets stage into scope for this test stage. diff --git a/eng/pipelines/pr/sqlclient-pr-pipeline.yml b/eng/pipelines/pr/sqlclient-pr-pipeline.yml index 5ad15f5474..640c448c13 100644 --- a/eng/pipelines/pr/sqlclient-pr-pipeline.yml +++ b/eng/pipelines/pr/sqlclient-pr-pipeline.yml @@ -122,14 +122,14 @@ stages: stageName: ${{ variables.stageNamePack }} packArtifactBaseName: ${{ variables.packArtifactBaseName }} poolName: $(PoolNameDefault) - vmImage: ADO-UB24 + poolImage: ADO-UB24 # Stage 1b: Generate secrets - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml@self parameters: stageName: ${{ variables.stageNameSecrets }} poolName: $(PoolNameDefault) - vmImage: ADO-UB24 + poolImage: ADO-UB24 # Stage 2: Execute tests and collect code coverage - template: /eng/pipelines/pr/stages/test-stages.yml@self @@ -160,7 +160,7 @@ stages: parameters: coverageArtifactBaseName: ${{ variables.coverageArtifactBaseName }} poolName: $(PoolNameDefault) - vmImage: ADO-UB24 + poolImage: ADO-UB24 dependsOn: - ${{ each platform in parameters.platforms }}: - "test_${{ platform.displayName }}" diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml index a81b4bb1be..9fa4c1828a 100644 --- a/eng/pipelines/pr/stages/collect-coverage-stage.yml +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -21,7 +21,7 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string stages: @@ -37,7 +37,7 @@ stages: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/pr/stages/generate-secrets-stage.yml b/eng/pipelines/pr/stages/generate-secrets-stage.yml index bd7d2d6a5c..f936594e1a 100644 --- a/eng/pipelines/pr/stages/generate-secrets-stage.yml +++ b/eng/pipelines/pr/stages/generate-secrets-stage.yml @@ -35,7 +35,7 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string stages: @@ -57,7 +57,7 @@ stages: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/pr/stages/pack-stage.yml b/eng/pipelines/pr/stages/pack-stage.yml index 8da436f445..a142eac5e9 100644 --- a/eng/pipelines/pr/stages/pack-stage.yml +++ b/eng/pipelines/pr/stages/pack-stage.yml @@ -47,7 +47,7 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string stages: @@ -63,7 +63,7 @@ stages: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: # Install dotnet @@ -147,7 +147,3 @@ stages: artifact: ${{ parameters.packArtifactBaseName }}_attempt$(System.JobAttempt) displayName: Publish Build Output condition: succeededOrFailed() - - - - diff --git a/eng/pipelines/pr/stages/test-stages.yml b/eng/pipelines/pr/stages/test-stages.yml index 0ad9921ec6..ce4c5c32f8 100644 --- a/eng/pipelines/pr/stages/test-stages.yml +++ b/eng/pipelines/pr/stages/test-stages.yml @@ -133,7 +133,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} @@ -148,7 +148,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} @@ -163,7 +163,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} @@ -178,7 +178,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} @@ -197,7 +197,7 @@ stages: platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} platformOperatingSystem: ${{ platform.operatingSystem }} poolName: ${{ parameters.poolName }} @@ -232,7 +232,7 @@ stages: platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} platformOperatingSystem: ${{ platform.operatingSystem }} poolName: ${{ parameters.poolName }} @@ -261,7 +261,7 @@ stages: platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} platformOperatingSystem: ${{ platform.operatingSystem }} poolName: ${{ parameters.poolName }} @@ -290,7 +290,7 @@ stages: platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} platformOperatingSystem: ${{ platform.operatingSystem }} poolName: ${{ parameters.poolName }} @@ -316,7 +316,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} diff --git a/eng/pipelines/stages/build-abstractions-package-ci-stage.yml b/eng/pipelines/stages/build-abstractions-package-ci-stage.yml index 3342bfca01..03fac7ae8e 100644 --- a/eng/pipelines/stages/build-abstractions-package-ci-stage.yml +++ b/eng/pipelines/stages/build-abstractions-package-ci-stage.yml @@ -119,7 +119,7 @@ stages: netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] poolName: ${{ parameters.poolName }} - vmImage: ADO-UB24 + poolImage: ADO-UB24 # ------------------------------------------------------------------------ # Build and test on Windows @@ -134,7 +134,7 @@ stages: netFrameworkRuntimes: [net462] netRuntimes: [net8.0, net9.0, net10.0] poolName: ${{ parameters.poolName }} - vmImage: ADO-Win25 + poolImage: ADO-Win25 # ------------------------------------------------------------------------ # Build and test on macOS. @@ -151,7 +151,7 @@ stages: # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted # 'Azure Pipelines' pool. poolName: Azure Pipelines - vmImage: macos-latest + poolImage: macos-latest # ------------------------------------------------------------------------ # Create and publish the NuGet package. @@ -159,6 +159,7 @@ stages: - template: /eng/pipelines/jobs/pack-abstractions-package-ci-job.yml@self parameters: poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} # The version is computed by this stage (see the packageVersion variable above). packageVersion: $(packageVersion) diff --git a/eng/pipelines/stages/build-azure-package-ci-stage.yml b/eng/pipelines/stages/build-azure-package-ci-stage.yml index fed69e339e..1f502a5fee 100644 --- a/eng/pipelines/stages/build-azure-package-ci-stage.yml +++ b/eng/pipelines/stages/build-azure-package-ci-stage.yml @@ -162,7 +162,7 @@ stages: - template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self parameters: saPassword: $(saPassword) - vmImage: ADO-UB24-SQL25 + poolImage: ADO-UB24-SQL25 # ------------------------------------------------------------------------ # Build and test on Windows @@ -197,7 +197,7 @@ stages: # group. fileStreamDirectory: $(FileStreamDirectory) SQLRootPath: $(SQL25RootPath) - vmImage: ADO-MMS25-SQL25 + poolImage: ADO-MMS25-SQL25 # ------------------------------------------------------------------------ # Build and test on macOS. @@ -222,7 +222,7 @@ stages: # 'Azure Pipelines' pool. poolName: Azure Pipelines referenceType: ${{ parameters.referenceType }} - vmImage: macos-latest + poolImage: macos-latest # ------------------------------------------------------------------------ # Create and publish the NuGet package. @@ -230,6 +230,7 @@ stages: - template: /eng/pipelines/jobs/pack-azure-package-ci-job.yml@self parameters: poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} packageVersion: $(packageVersion) azureArtifactsName: ${{ parameters.azureArtifactsName }} diff --git a/eng/pipelines/stages/build-logging-package-ci-stage.yml b/eng/pipelines/stages/build-logging-package-ci-stage.yml index 9b421b821a..62b52480c9 100644 --- a/eng/pipelines/stages/build-logging-package-ci-stage.yml +++ b/eng/pipelines/stages/build-logging-package-ci-stage.yml @@ -93,6 +93,7 @@ stages: - template: /eng/pipelines/jobs/pack-logging-package-ci-job.yml@self parameters: poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 loggingArtifactsName: ${{ parameters.loggingArtifactsName }} # The version is computed by this stage (see the packageVersion variable above). packageVersion: $(packageVersion) diff --git a/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml b/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml index 1895f4e593..f7eb9d6f1a 100644 --- a/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml +++ b/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml @@ -93,6 +93,7 @@ stages: - template: /eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml@self parameters: poolName: ${{ parameters.poolName }} + poolImage: ADO-Win25 buildConfiguration: ${{ parameters.buildConfiguration }} referenceType: ${{ parameters.referenceType }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} diff --git a/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml b/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml index 633e3bc3f1..f62a6eb166 100644 --- a/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml +++ b/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml @@ -86,6 +86,7 @@ stages: - template: /eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml@self parameters: poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} diff --git a/eng/pipelines/stages/compute-versions-ci-stage.yml b/eng/pipelines/stages/compute-versions-ci-stage.yml index d508ba51ed..ada6a8844b 100644 --- a/eng/pipelines/stages/compute-versions-ci-stage.yml +++ b/eng/pipelines/stages/compute-versions-ci-stage.yml @@ -28,9 +28,8 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string - default: ADO-UB24 # Build suffix appended to prerelease tag (e.g. 'ci' or 'pr'). - name: buildSuffix @@ -47,7 +46,7 @@ stages: pool: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: # Install the global.json-pinned .NET SDK before running any dotnet build, so this stage diff --git a/eng/pipelines/stages/generate-secrets-ci-stage.yml b/eng/pipelines/stages/generate-secrets-ci-stage.yml index d6f38f8f85..5e54a5d1b1 100644 --- a/eng/pipelines/stages/generate-secrets-ci-stage.yml +++ b/eng/pipelines/stages/generate-secrets-ci-stage.yml @@ -40,9 +40,8 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string - default: ADO-UB24 stages: @@ -65,7 +64,7 @@ stages: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml b/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml index 06d43769fd..d16b4a5838 100644 --- a/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml +++ b/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml @@ -47,9 +47,8 @@ parameters: type: string # The name of the VM image to run on, within the pool. - - name: vmImage + - name: poolImage type: string - default: ADO-Win25 stages: @@ -71,7 +70,7 @@ stages: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: # The directory where all package artifacts will be downloaded. From fde91ee8af60b7c7fc40fb4edbf632998bdd6d55 Mon Sep 17 00:00:00 2001 From: priyankatiwari08 Date: Thu, 10 Sep 2026 21:25:21 +0530 Subject: [PATCH 34/51] Use automatic model selection in issue auto-triage workflow (#4659) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .gitattributes | 2 +- .github/aw/actions-lock.json | 6 +- .github/workflows/issue-triage.lock.yml | 1033 ++++++++++++++--------- .github/workflows/issue-triage.md | 2 + .github/workflows/verify-aw-lock.yml | 4 +- 5 files changed, 640 insertions(+), 407 deletions(-) diff --git a/.gitattributes b/.gitattributes index a1d79dfeee..ab4f136a23 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,4 +2,4 @@ * text=auto # Treat workflow lock files as generated -.github/workflows/*.lock.yml linguist-generated=true merge=ours +.github/workflows/*.lock.yml linguist-generated=true diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 64a0a0a93a..cdcb39f6a2 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -5,10 +5,10 @@ "version": "v9.0.0", "sha": "3a2844b7e9c422d3c10d287c895573f7108da1b3" }, - "github/gh-aw-actions/setup@v0.80.9": { + "github/gh-aw-actions/setup@v0.88.2": { "repo": "github/gh-aw-actions/setup", - "version": "v0.80.9", - "sha": "8c7d04ebf1ece56cd381446125da3e0f6896294a" + "version": "v0.88.2", + "sha": "9271a1804551c0dc4fb0085a97979950aa2f8489" } } } diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml index 886e8e2e63..d1ef3b2e67 100644 --- a/.github/workflows/issue-triage.lock.yml +++ b/.github/workflows/issue-triage.lock.yml @@ -1,15 +1,15 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"c9a288bea873cc74dd33e81e0f5784e167d990ee0e2645c1722106b45ae1f77d","body_hash":"99b21e9ba167d3bfbcfaeb3b04c515fc42b56679f0ee175c119bc5843f034d31","compiler_version":"v0.80.9","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.63"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"8c7d04ebf1ece56cd381446125da3e0f6896294a","version":"v0.80.9"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.7","digest":"sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7","digest":"sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.7","digest":"sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.27","digest":"sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.27@sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.4.0","digest":"sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036","pinned_image":"ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036"}]} -# This file was automatically generated by gh-aw (v0.80.9). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8b81a4840372af279438b0250d6ae7168fa2e0a2f82ae8cd52d714f18a83d425","body_hash":"99b21e9ba167d3bfbcfaeb3b04c515fc42b56679f0ee175c119bc5843f034d31","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","missing_data","missing_tool","noop","remove_labels"]}]} +# This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # -# ___ _ _ -# / _ \ | | (_) -# | |_| | __ _ ___ _ __ | |_ _ ___ +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ # | _ |/ _` |/ _ \ '_ \| __| |/ __| -# | | | | (_| | __/ | | | |_| | (__ +# | | | | (_| | __/ | | | |_| | (__ # \_| |_/\__, |\___|_| |_|\__|_|\___| # __/ | -# _ _ |___/ +# _ _ |___/ # | | | | / _| | # | | | | ___ _ __ _ __| |_| | _____ ____ # | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| @@ -26,45 +26,55 @@ # # Secrets used: # - COPILOT_GITHUB_TOKEN +# - GH_AW_DEFAULT_OTLP_HEADERS # - GH_AW_GITHUB_MCP_SERVER_TOKEN # - GH_AW_GITHUB_TOKEN # - GITHUB_TOKEN # # Custom actions used: -# - actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 -# - actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 -# - actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 +# - actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 +# - actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 +# - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 -# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 -# - github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 +# - github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 # # Container images used: -# - ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c -# - ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6 -# - ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96 -# - ghcr.io/github/gh-aw-mcpg:v0.3.27@sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7 -# - ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b -# - ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036 +# - ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 +# - ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f +# - ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e +# - ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e +# - ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699 name: "SqlClient Issue Auto-Triage" on: issue_comment: types: - - created + - created issues: types: - - opened - # roles: all # Roles processed as role check in pre-activation job + - opened +# roles: all # Roles processed as role check in pre-activation job permissions: {} concurrency: group: "gh-aw-${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}" + queue: max run-name: "SqlClient Issue Auto-Triage" +env: + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.GH_AW_DEFAULT_OTLP_ENDPOINT }} + OTEL_SERVICE_NAME: gh-aw.issue-triage + OTEL_RESOURCE_ATTRIBUTES: 'gh-aw.workflow.name=SqlClient%20Issue%20Auto-Triage,gh-aw.repository=${{ github.repository }},gh-aw.run.id=${{ github.run_id }},github.run_id=${{ github.run_id }},gh-aw.engine.id=copilot' + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.GH_AW_DEFAULT_OTLP_HEADERS }} + GH_AW_OTLP_ENDPOINTS: '[{"url":"${{ vars.GH_AW_DEFAULT_OTLP_ENDPOINT }}","headers":"${{ secrets.GH_AW_DEFAULT_OTLP_HEADERS }}"}]' + GH_AW_OTLP_IF_MISSING: ignore + jobs: activation: if: > @@ -87,16 +97,19 @@ jobs: contents: read env: GH_AW_MAX_DAILY_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_DAILY_AI_CREDITS || '5000' }} + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} outputs: body: ${{ steps.sanitized.outputs.body }} comment_id: "" comment_repo: "" daily_ai_credits_exceeded: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_exceeded == 'true' }} + daily_ai_credits_guardrail_status: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_guardrail_status || '' }} daily_ai_credits_threshold: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_threshold || '' }} daily_ai_credits_total_effective_tokens: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_total_effective_tokens || '' }} engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} model: ${{ steps.generate_aw_info.outputs.model }} + oauth_token_check_failed: ${{ steps.check-oauth-tokens.outputs.oauth_token_check_failed == 'true' }} setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} @@ -106,7 +119,7 @@ jobs: steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -114,40 +127,45 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" - name: Generate agentic run info id: generate_aw_info env: GH_AW_INFO_ENGINE_ID: "copilot" GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AGENT_VERSION: "1.0.63" - GH_AW_INFO_CLI_VERSION: "v0.80.9" + GH_AW_INFO_MODEL: "auto" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AGENT_VERSION: "1.0.80" + GH_AW_INFO_CLI_VERSION: "v0.88.2" GH_AW_INFO_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_INFO_EXPERIMENTAL: "false" GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" GH_AW_INFO_STAGED: "false" GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' GH_AW_INFO_FIREWALL_ENABLED: "true" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_AWMG_VERSION: "" GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_AGENT_RUNTIME: "" GH_AW_COMPILED_STRICT: "true" uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + const { main } = require(path.join(actionsDir, 'generate_aw_info.cjs')); await main(core, context); - name: Restore daily AIC usage cache id: restore-daily-aic-cache if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} continue-on-error: true - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: key: agentic-workflow-usage-issuetriage-${{ github.run_id }} restore-keys: agentic-workflow-usage-issuetriage- @@ -163,9 +181,11 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/restore_aic_usage_cache_fallback.cjs'); + const { main } = require(path.join(actionsDir, 'restore_aic_usage_cache_fallback.cjs')); await main(); - name: Check daily workflow token guardrail id: daily-effective-workflow-guardrail @@ -183,32 +203,38 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_daily_aic_workflow_guardrail.cjs'); + const { main } = require(path.join(actionsDir, 'check_daily_aic_workflow_guardrail.cjs')); await main(); + - name: Check for OAuth tokens + id: check-oauth-tokens + run: bash "${RUNNER_TEMP}/gh-aw/actions/check_oauth_tokens.sh" + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} - name: Checkout .github and .agents folders - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: | .github .agents - .antigravity .claude .codex - .crush .gemini - .opencode .pi sparse-checkout-cone-mode: true fetch-depth: 1 - name: Save agent config folders for base branch restoration env: - GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" - GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + GH_AW_AGENT_FOLDERS: ".agents .github" + GH_AW_AGENT_FILES: "AGENTS.md" + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" - name: Check workflow lock file id: check-lock-file uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -217,35 +243,47 @@ jobs: GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + const { main } = require(path.join(actionsDir, 'check_workflow_timestamp_api.cjs')); await main(); - name: Check compile-agentic version uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_COMPILED_VERSION: "v0.80.9" + GH_AW_COMPILED_VERSION: "v0.88.2" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); + const { main } = require(path.join(actionsDir, 'check_version_updates.cjs')); await main(); - name: Compute current body text id: sanitized uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/compute_text.cjs'); + const { main } = require(path.join(actionsDir, 'compute_text.cjs')); await main(); + - name: Log runtime features + if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh" - name: Create prompt with built-in context + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ACTIONS_DIR: ${{ runner.temp }}/gh-aw/actions + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"file\":\"mcp_cli_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"file\":\"github_mcp_tools_with_safeoutputs_prompt.md\"},{\"file\":\"pr_context_prompt.md\",\"condition_env\":\"GH_AW_INCLUDE_PR_CONTEXT\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"}]}" GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} @@ -254,78 +292,36 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_IS_PR_COMMENT: ${{ github.event.issue.pull_request && 'true' || '' }} - # poutine:ignore untrusted_checkout_exec - run: | - bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" - { - cat << 'GH_AW_PROMPT_09a4b5a8a8b33a81_EOF' - - GH_AW_PROMPT_09a4b5a8a8b33a81_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_09a4b5a8a8b33a81_EOF' - - Tools: add_comment, add_labels, remove_labels, missing_tool, missing_data, noop - - GH_AW_PROMPT_09a4b5a8a8b33a81_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_09a4b5a8a8b33a81_EOF' - - The following GitHub context information is available for this workflow: - {{#if github.actor}} - - **actor**: __GH_AW_GITHUB_ACTOR__ - {{/if}} - {{#if github.repository}} - - **repository**: __GH_AW_GITHUB_REPOSITORY__ - {{/if}} - {{#if github.workspace}} - - **workspace**: __GH_AW_GITHUB_WORKSPACE__ - {{/if}} - {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} - - **issue-number**: #__GH_AW_EXPR_802A9F6A__ - {{/if}} - {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} - - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ - {{/if}} - {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} - - **pull-request-number**: #__GH_AW_EXPR_463A214A__ - {{/if}} - {{#if github.event.comment.id || github.aw.context.comment_id}} - - **comment-id**: __GH_AW_EXPR_FF1D34CE__ - {{/if}} - {{#if github.run_id}} - - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ - {{/if}} - - - GH_AW_PROMPT_09a4b5a8a8b33a81_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - if [ "$GITHUB_EVENT_NAME" = "issue_comment" ] && [ -n "$GH_AW_IS_PR_COMMENT" ] || [ "$GITHUB_EVENT_NAME" = "pull_request_review_comment" ] || [ "$GITHUB_EVENT_NAME" = "pull_request_review" ]; then - cat "${RUNNER_TEMP}/gh-aw/prompts/pr_context_prompt.md" - fi - cat << 'GH_AW_PROMPT_09a4b5a8a8b33a81_EOF' - - {{#runtime-import .github/workflows/issue-triage.md}} - GH_AW_PROMPT_09a4b5a8a8b33a81_EOF - } > "$GH_AW_PROMPT" + GH_AW_INCLUDE_PR_CONTEXT: ${{ (github.event_name == 'issue_comment' && github.event.issue.pull_request != null) || github.event_name == 'pull_request_review_comment' || github.event_name == 'pull_request_review' }} + GH_AW_PROMPT_CONTENT_0000: "\n" + GH_AW_PROMPT_CONTENT_0001: "\nTools: add_comment, add_labels, remove_labels, missing_tool, missing_data, noop\n" + GH_AW_PROMPT_CONTENT_0002: "\n" + GH_AW_PROMPT_CONTENT_0003: "\nThe following GitHub context information is available for this workflow:\n{{#if github.actor}}\n- **actor**: __GH_AW_GITHUB_ACTOR__\n{{/if}}\n{{#if github.repository}}\n- **repository**: __GH_AW_GITHUB_REPOSITORY__\n{{/if}}\n{{#if github.workspace}}\n- **workspace**: __GH_AW_GITHUB_WORKSPACE__\n{{/if}}\n{{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}\n- **issue-number**: #__GH_AW_EXPR_802A9F6A__\n{{/if}}\n{{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}\n- **discussion-number**: #__GH_AW_EXPR_1A3A194A__\n{{/if}}\n{{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}\n- **pull-request-number**: #__GH_AW_EXPR_463A214A__\n{{/if}}\n{{#if github.event.comment.id || github.aw.context.comment_id}}\n- **comment-id**: __GH_AW_EXPR_FF1D34CE__\n{{/if}}\n{{#if github.run_id}}\n- **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__\n{{/if}}\n\n\n" + GH_AW_PROMPT_CONTENT_0004: "\n" + GH_AW_PROMPT_CONTENT_0005: "{{#runtime-import .github/workflows/issue-triage.md}}\n" + with: + script: | + const { setupGlobals } = require(process.env.GH_AW_ACTIONS_DIR + '/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(process.env.GH_AW_ACTIONS_DIR + '/create_prompt.cjs'); + await main(core); - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_ENGINE_ID: "copilot" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); + const { main } = require(path.join(actionsDir, 'interpolate_prompt.cjs')); await main(); - name: Substitute placeholders uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} @@ -334,15 +330,17 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_IS_PR_COMMENT: ${{ github.event.issue.pull_request && 'true' || '' }} - GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + GH_AW_INCLUDE_PR_CONTEXT: ${{ (github.event_name == 'issue_comment' && github.event.issue.pull_request != null) || github.event_name == 'pull_request_review_comment' || github.event_name == 'pull_request_review' }} + GH_AW_MCP_CLI_SERVERS_LIST: "- `github` — run `github --help` to see available tools\n- `safeoutputs` — run `safeoutputs --help` to see available tools" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - - const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); - + + const substitutePlaceholders = require(path.join(actionsDir, 'substitute_placeholders.cjs')); + // Call the substitution function return await substitutePlaceholders({ file: process.env.GH_AW_PROMPT, @@ -355,22 +353,26 @@ jobs: GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, - GH_AW_IS_PR_COMMENT: process.env.GH_AW_IS_PR_COMMENT, + GH_AW_INCLUDE_PR_CONTEXT: process.env.GH_AW_INCLUDE_PR_CONTEXT, GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST } }); - name: Validate prompt placeholders env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" - name: Print prompt env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Stage prompt files for artifact upload + run: | + mkdir -p /tmp/gh-aw/aw-prompts + cp -a "${RUNNER_TEMP}/gh-aw/aw-prompts/." /tmp/gh-aw/aw-prompts/ - name: Upload activation artifact - if: success() + if: success() || failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: activation @@ -397,12 +399,20 @@ jobs: contents: read issues: read pull-requests: read + timeout-minutes: 60 env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_AW_ASSETS_ALLOWED_EXTS: "" GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_PR_HEAD_BASE_BRANCH: "" + GH_AW_PR_HEAD_BASE_PR_NUMBER: "" + GH_AW_PR_HEAD_BASE_REF: "" + GH_AW_PR_HEAD_BASE_REPO: "" + GH_AW_PR_HEAD_BASE_SHA: "" + GH_AW_PR_HEAD_REPO: "" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} GH_AW_WORKFLOW_ID_SANITIZED: issuetriage outputs: agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} @@ -412,8 +422,13 @@ jobs: checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} has_patch: ${{ steps.collect_output.outputs.has_patch }} + http_400_response_error: ${{ steps.detect-agent-errors.outputs.http_400_response_error || 'false' }} inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + invocation_cap_exceeded: ${{ steps.detect-agent-errors.outputs.invocation_cap_exceeded || 'false' }} + max_cache_misses_exceeded: ${{ steps.detect-agent-errors.outputs.max_cache_misses_exceeded || 'false' }} mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + missing_model_pricing_error: ${{ steps.detect-agent-errors.outputs.missing_model_pricing_error || 'false' }} + missing_model_pricing_model_name: ${{ steps.detect-agent-errors.outputs.missing_model_pricing_model_name || '' }} model: ${{ needs.activation.outputs.model }} model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} output: ${{ steps.collect_output.outputs.output }} @@ -421,11 +436,12 @@ jobs: setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} + shell_expansion_guard_rejected: ${{ steps.detect-agent-errors.outputs.shell_expansion_guard_rejected || 'false' }} unknown_model_ai_credits: ${{ steps.parse-mcp-gateway.outputs.unknown_model_ai_credits || 'false' }} steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -434,19 +450,28 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" - name: Set runtime paths id: set-runtime-paths + env: + GH_AW_RUNNER_TOOL_CACHE: ${{ runner.tool_cache }} run: | + if [ -z "${RUNNER_TOOL_CACHE:-}" ]; then + echo "RUNNER_TOOL_CACHE=${GH_AW_RUNNER_TOOL_CACHE}" >> "$GITHUB_ENV" + fi { echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" } >> "$GITHUB_OUTPUT" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" + - name: Check OTLP telemetry configuration + run: bash "${RUNNER_TEMP}/gh-aw/actions/check_otlp_default_credentials.sh" - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Create gh-aw temp directory @@ -455,6 +480,11 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -471,16 +501,32 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs'); + const { main } = require(path.join(actionsDir, 'checkout_pr_branch.cjs')); await main(); - name: Install GitHub Copilot CLI - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.63 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" env: GH_HOST: github.com + GH_AW_COMPILED_VERSION: v0.88.2 - name: Install AWF binary - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.7 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.12 --rootless + - name: Determine automatic lockdown mode for GitHub MCP Server + id: determine-automatic-lockdown + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + GH_AW_GITHUB_MIN_INTEGRITY: 'none' + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const determineAutomaticLockdown = require(path.join(actionsDir, 'determine_automatic_lockdown.cjs')); + await determineAutomaticLockdown(github, context, core); - name: Parse integrity filter lists id: parse-guard-vars env: @@ -488,16 +534,11 @@ jobs: GH_AW_TRUSTED_USERS_VAR: ${{ vars.GH_AW_GITHUB_TRUSTED_USERS || '' }} GH_AW_APPROVAL_LABELS_VAR: ${{ vars.GH_AW_GITHUB_APPROVAL_LABELS || '' }} run: bash "${RUNNER_TEMP}/gh-aw/actions/parse_guard_list.sh" - - name: Download activation artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: activation - path: /tmp/gh-aw - name: Restore agent config folders from base branch if: steps.checkout-pr.outcome == 'success' env: - GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" - GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + GH_AW_AGENT_FOLDERS: ".agents .github" + GH_AW_AGENT_FILES: "AGENTS.md" run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" - name: Restore inline sub-agents from activation artifact env: @@ -509,15 +550,26 @@ jobs: GH_AW_SKILL_DIR: ".github/skills" run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6 ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96 ghcr.io/github/gh-aw-mcpg:v0.3.27@sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7 ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036 - - name: Generate Safe Outputs Config + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699 + - name: Prepare Safe Outputs Directories run: | mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_723da9f6aa534eb7_EOF' - {"add_comment":{"hide_older_comments":true,"max":1},"add_labels":{"allowed":["Auto-Triage: Waiting for Author"],"max":1},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"remove_labels":{"allowed":["Auto-Triage: Waiting for Author"],"max":1},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_723da9f6aa534eb7_EOF + - name: Generate Safe Outputs Config + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_FILE_ROOT: "${{ runner.temp }}/gh-aw" + GH_AW_FILE_CONFIG: "{\"files\":[{\"path\":\"safeoutputs/config.json\",\"content_env\":\"GH_AW_SAFE_OUTPUTS_CONFIG\"}]}" + GH_AW_SAFE_OUTPUTS_CONFIG: "{\"add_comment\":{\"hide_older_comments\":true,\"max\":1},\"add_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"remove_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"report_incomplete\":{}}" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'create_files.cjs')); + await main(); - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -541,9 +593,18 @@ jobs: "sanitize": true, "maxLength": 65000 }, + "comment_id": { + "optionalPositiveInteger": true + }, "item_number": { "issueOrPRNumber": true }, + "pr": { + "issueOrPRNumber": true + }, + "pr_number": { + "issueOrPRNumber": true + }, "reply_to_id": { "type": "string", "maxLength": 256 @@ -551,6 +612,16 @@ jobs: "repo": { "type": "string", "maxLength": 256 + }, + "target": { + "type": "string", + "enum": [ + "status" + ] + }, + "temporary_id": { + "type": "string", + "pattern": "^#?aw_[A-Za-z0-9_]{3,12}$" } } }, @@ -562,10 +633,7 @@ jobs: }, "labels": { "required": true, - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 + "type": "array" }, "repo": { "type": "string", @@ -638,10 +706,7 @@ jobs: }, "labels": { "required": true, - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 + "type": "array" }, "repo": { "type": "string", @@ -669,56 +734,68 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + const { main } = require(path.join(actionsDir, 'generate_safe_outputs_tools.cjs')); await main(); - name: Start MCP Gateway id: start-mcp-gateway env: + GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST: ${{ vars.GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST || 'true' }} GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }} GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }} + GH_AW_SINK_VISIBILITY: ${{ steps.determine-automatic-lockdown.outputs.visibility }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eo pipefail mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" - + if [ -n "${GITHUB_EVENT_PATH:-}" ] && [ -r "${GITHUB_EVENT_PATH}" ]; then + GH_AW_SAFEOUTPUTS_EVENT_PATH="${RUNNER_TEMP}/gh-aw/safeoutputs/github_event.json" + cp "${GITHUB_EVENT_PATH}" "${GH_AW_SAFEOUTPUTS_EVENT_PATH}" + export GITHUB_EVENT_PATH="${GH_AW_SAFEOUTPUTS_EVENT_PATH}" + fi + # Export gateway environment variables for MCP config and gateway script export MCP_GATEWAY_PORT="8080" - export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_DOMAIN="awmg-mcpg" export MCP_GATEWAY_HOST_DOMAIN="localhost" - MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') - echo "::add-mask::${MCP_GATEWAY_API_KEY}" - export MCP_GATEWAY_API_KEY + MCP_GATEWAY_AGENT_ID=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_AGENT_ID}" + export MCP_GATEWAY_AGENT_ID export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" + export MCP_GATEWAY_ALLOWED_MOUNT_ROOTS="${GITHUB_WORKSPACE}:rw,${RUNNER_TEMP}/gh-aw:ro,${RUNNER_TEMP}/gh-aw/safeoutputs:rw,/opt:ro,/tmp:rw,/usr/bin/gh:ro" + export GH_AW_PR_HEAD_BASE_BRANCH="${GH_AW_PR_HEAD_BASE_BRANCH:-}" + export GH_AW_PR_HEAD_BASE_SHA="${GH_AW_PR_HEAD_BASE_SHA:-}" + export GH_AW_PR_HEAD_BASE_REPO="${GH_AW_PR_HEAD_BASE_REPO:-}" + export GH_AW_PR_HEAD_BASE_PR_NUMBER="${GH_AW_PR_HEAD_BASE_PR_NUMBER:-}" + export GH_AW_PR_HEAD_BASE_REF="${GH_AW_PR_HEAD_BASE_REF:-}" + export GH_AW_PR_HEAD_REPO="${GH_AW_PR_HEAD_REPO:-}" export DEBUG="*" - + export GH_AW_ENGINE="copilot" MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') - case "${DOCKER_HOST:-}" in - unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; - /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; - * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; - esac - DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --name awmg-mcpg --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.3.27' - + source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_AGENT_ID -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_PR_HEAD_BASE_BRANCH -e GH_AW_PR_HEAD_BASE_SHA -e GH_AW_PR_HEAD_BASE_REPO -e GH_AW_PR_HEAD_BASE_PR_NUMBER -e GH_AW_PR_HEAD_BASE_REF -e GH_AW_PR_HEAD_REPO -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e RUNNER_TOOL_CACHE -e MCP_GATEWAY_ALLOWED_MOUNT_ROOTS -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.15' + mkdir -p "$HOME/.copilot" GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_4b806ec8e0233008_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_137b94c134aafdc9_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v1.4.0", + "container": "ghcr.io/github/github-mcp-server:v1.11.0", "env": { - "GITHUB_HOST": "\${GITHUB_SERVER_URL}", - "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_FEATURES": "fields_param", + "GITHUB_HOST": "${GITHUB_SERVER_URL}", + "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" }, @@ -749,7 +826,17 @@ jobs: "GH_AW_SAFE_OUTPUTS": "\${GH_AW_SAFE_OUTPUTS}", "GH_AW_SAFE_OUTPUTS_CONFIG_PATH": "\${GH_AW_SAFE_OUTPUTS_CONFIG_PATH}", "GH_AW_SAFE_OUTPUTS_TOOLS_PATH": "\${GH_AW_SAFE_OUTPUTS_TOOLS_PATH}", + "GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST": "\${GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST}", + "GH_AW_PR_HEAD_BASE_BRANCH": "\${GH_AW_PR_HEAD_BASE_BRANCH}", + "GH_AW_PR_HEAD_BASE_SHA": "\${GH_AW_PR_HEAD_BASE_SHA}", + "GH_AW_PR_HEAD_BASE_REPO": "\${GH_AW_PR_HEAD_BASE_REPO}", + "GH_AW_PR_HEAD_BASE_PR_NUMBER": "\${GH_AW_PR_HEAD_BASE_PR_NUMBER}", + "GH_AW_PR_HEAD_BASE_REF": "\${GH_AW_PR_HEAD_BASE_REF}", + "GH_AW_PR_HEAD_REPO": "\${GH_AW_PR_HEAD_REPO}", + "GITHUB_EVENT_NAME": "\${GITHUB_EVENT_NAME}", + "GITHUB_EVENT_PATH": "\${GITHUB_EVENT_PATH}", "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}", + "GITHUB_SHA": "\${GITHUB_SHA}", "GITHUB_TOKEN": "\${GITHUB_TOKEN}", "GITHUB_WORKSPACE": "\${GITHUB_WORKSPACE}", "RUNNER_TEMP": "\${RUNNER_TEMP}" @@ -758,7 +845,8 @@ jobs: "write-sink": { "accept": [ "*" - ] + ], + "sink-visibility": "${GH_AW_SINK_VISIBILITY}" } } } @@ -766,24 +854,32 @@ jobs: "gateway": { "port": $MCP_GATEWAY_PORT, "domain": "${MCP_GATEWAY_DOMAIN}", - "apiKey": "${MCP_GATEWAY_API_KEY}", - "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + "agentId": "${MCP_GATEWAY_AGENT_ID}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}", + "startupTimeout": 120, + "opentelemetry": { + "endpoint": "${OTEL_EXPORTER_OTLP_ENDPOINT}", + "traceId": "${GITHUB_AW_OTEL_TRACE_ID}", + "spanId": "${GITHUB_AW_OTEL_PARENT_SPAN_ID}" + } } } - GH_AW_MCP_CONFIG_4b806ec8e0233008_EOF + GH_AW_MCP_CONFIG_137b94c134aafdc9_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true env: - MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_AGENT_ID: ${{ steps.start-mcp-gateway.outputs.gateway-agent-id }} MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io); - const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); + const { main } = require(path.join(actionsDir, 'mount_mcp_as_cli.cjs')); await main(); - name: Clean credentials continue-on-error: true @@ -795,44 +891,64 @@ jobs: - name: Execute GitHub Copilot CLI id: agentic_execution # Copilot CLI tool arguments (sorted): - timeout-minutes: 20 + # --allow-tool github + # --allow-tool safeoutputs + # --allow-tool shell(cat) + # --allow-tool shell(date) + # --allow-tool shell(echo) + # --allow-tool shell(find) + # --allow-tool shell(github:*) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(ls) + # --allow-tool shell(printf) + # --allow-tool shell(pwd) + # --allow-tool shell(safeoutputs:*) + # --allow-tool shell(sort) + # --allow-tool shell(tail) + # --allow-tool shell(uniq) + # --allow-tool shell(wc) + # --allow-tool shell(yq) + # --allow-tool write + timeout-minutes: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} run: | set -o pipefail printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt - trap 'rm -f "$HOME/.copilot/settings.json"' EXIT + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"; if [ "$gh_aw_exit_code" -ne 0 ]; then echo "::error::Agent execution exited with code $gh_aw_exit_code"; fi' EXIT mkdir -p "$HOME/.copilot" printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" export XDG_CONFIG_HOME="$HOME" export GH_AW_MCP_CONFIG="$HOME/.copilot/mcp-config.json" + GH_AW_COPILOT_SRC="$(command -v copilot 2>/dev/null || true)" + if [ -z "$GH_AW_COPILOT_SRC" ] || [ ! -x "$GH_AW_COPILOT_SRC" ]; then + echo "GitHub Copilot CLI executable not found on PATH after installation" >&2 + exit 127 + fi + GH_AW_COPILOT_BIN="${RUNNER_TEMP}/gh-aw/bin/copilot" + mkdir -p "${RUNNER_TEMP}/gh-aw/bin" + if [ "$GH_AW_COPILOT_SRC" != "$GH_AW_COPILOT_BIN" ]; then + cp "$GH_AW_COPILOT_SRC" "$GH_AW_COPILOT_BIN" + fi + chmod 755 "$GH_AW_COPILOT_BIN" + touch /tmp/gh-aw/agent-step-summary.md GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) export GH_AW_NODE_BIN export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" (umask 177 && touch /tmp/gh-aw/agent-stdio.log) GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-1000}" - printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.7/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"github.com\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"ppa.launchpad.net\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"telemetry.enterprise.githubcopilot.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.5\",\"gpt-5.4\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.7,squid=sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96,agent=sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c,api-proxy=sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6,cli-proxy=sha256:4757f198a3fa20f88bdbe70be7ae1a05f127d9c0a9e96a5d6460ef40c08fc83d\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + if [[ ! "$GH_AW_MAX_AI_CREDITS" =~ ^[0-9]+$ ]]; then + GH_AW_MAX_AI_CREDITS="1000" + fi + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.28.12/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"ppa.launchpad.net\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"],\"isolation\":true,\"topologyAttach\":[\"awmg-mcpg\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-3.7-flash\":[\"copilot/gemini-3.7*flash*\",\"google/gemini-3.7*flash*\",\"gemini/gemini-3.7*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.28.12,squid=sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f,agent=sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202,api-proxy=sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32,cli-proxy=sha256:5250629d48eaedfedf2e948785228e8da29eec2a83cbab58ea0751c14a7b021d\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" GH_AW_DOCKER_HOST="" if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then GH_AW_DOCKER_HOST="${DOCKER_HOST}" fi - GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then - GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" - python3 - <<'PY' - import json,os,subprocess as sp - from pathlib import Path - try: - p=Path(os.environ["RUNNER_TEMP"])/"gh-aw"/"awf-config.json" - c=json.loads(p.read_text()) - c["chroot"]={"binariesSourcePath":"/tmp/gh-aw","identity":{"user":sp.check_output(["id","-un"],text=True).strip(),"uid":int(sp.check_output(["id","-u"],text=True)),"gid":int(sp.check_output(["id","-g"],text=True)),"home":"/tmp/gh-aw/home"}} - out=json.dumps(c,separators=(",",":"),ensure_ascii=False)+"\n" - p.write_text(out) - Path("/tmp/gh-aw/awf-config.json").write_text(out) - except Exception as e: - raise SystemExit(f"chroot config patch failed: {e}") from e - PY + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" fi GH_AW_TOOL_CACHE_MOUNT="" GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" @@ -841,22 +957,28 @@ jobs: GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" fi fi - # shellcheck disable=SC1003,SC2086 - sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ - -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + # shellcheck disable=SC1003,SC2016,SC2086 + GH_AW_AWF_ENGINE_NAME=copilot \ + GH_AW_AWF_HARNESS_MARKER='[copilot-harness]' \ + GH_AW_AWF_LOG_FILE=/tmp/gh-aw/agent-stdio.log \ + GH_AW_AWF_ATTEMPT_LOG_NAME=copilot \ + bash "${RUNNER_TEMP}/gh-aw/actions/run_awf_with_startup_retries.sh" -- \ + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_AGENT_ID --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + COPILOT_MODEL: auto + GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_TIMEOUT_MINUTES: 20 - GH_AW_VERSION: v0.80.9 + GH_AW_TIMEOUT_MINUTES: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} + GH_AW_VERSION: v0.88.2 GITHUB_API_URL: ${{ github.api_url }} GITHUB_AW: true GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows @@ -876,7 +998,18 @@ jobs: if: always() id: detect-agent-errors continue-on-error: true - run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" + env: + GH_AW_AGENTIC_EXECUTION_OUTCOME: ${{ steps.agentic_execution.outcome }} + GH_AW_ENGINE_STEP_TIMEOUT_MINUTES: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'detect_agent_errors.cjs')); + await main(); - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -892,7 +1025,7 @@ jobs: continue-on-error: true env: MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} - MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_AGENT_ID: ${{ steps.start-mcp-gateway.outputs.gateway-agent-id }} GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} run: | bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" @@ -901,9 +1034,11 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); + const { main } = require(path.join(actionsDir, 'redact_secrets.cjs')); await main(); env: GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' @@ -927,25 +1062,30 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); + const { main } = require(path.join(actionsDir, 'collect_ndjson_output.cjs')); await main(); - name: Parse agent logs for step summary if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); + const { main } = require(path.join(actionsDir, 'parse_copilot_log.cjs')); await main(); - name: Parse MCP Gateway logs for step summary if: always() @@ -953,34 +1093,29 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + const { main } = require(path.join(actionsDir, 'parse_mcp_gateway_log.cjs')); await main(); - name: Print firewall logs if: always() continue-on-error: true env: AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs - run: | - # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts - # AWF runs with sudo, creating files owned by root - sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true - # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) - if command -v awf &> /dev/null; then - awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" - else - echo 'AWF binary not installed, skipping firewall log summary' - fi + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_firewall_logs.sh" --rootless - name: Parse token usage for step summary if: always() continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + const { main } = require(path.join(actionsDir, 'parse_token_usage.cjs')); await main(); - name: Print AWF reflect summary if: always() @@ -988,16 +1123,41 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + const { main } = require(path.join(actionsDir, 'awf_reflect_summary.cjs')); await main(); + - name: Generate observability summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'generate_observability_summary.cjs')); + await main(core); - name: Write agent output placeholder if missing if: always() run: | if [ ! -f /tmp/gh-aw/agent_output.json ]; then echo '{"items":[]}' > /tmp/gh-aw/agent_output.json fi + # Small dedicated copy of the agent output so safe-output processing + # survives a failed or timed-out upload of the larger agent artifact + - name: Upload agent output fallback artifact + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent-output-fallback + path: | + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/safeoutputs.jsonl + if-no-files-found: ignore - name: Upload agent artifacts if: always() continue-on-error: true @@ -1016,6 +1176,8 @@ jobs: /tmp/gh-aw/pre-agent-audit.txt /tmp/gh-aw/agent/ /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/otel.jsonl + /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json /tmp/gh-aw/aw-*.patch @@ -1034,17 +1196,20 @@ jobs: - safe_outputs if: > always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || - needs.activation.outputs.stale_lock_file_failed == 'true' || needs.activation.outputs.daily_ai_credits_exceeded == 'true') + needs.activation.outputs.oauth_token_check_failed == 'true' || needs.activation.outputs.stale_lock_file_failed == 'true' || + needs.activation.outputs.daily_ai_credits_exceeded == 'true') runs-on: ubuntu-slim environment: issue-triage permissions: - contents: read + actions: read issues: write pull-requests: write concurrency: group: "gh-aw-conclusion-issue-triage" cancel-in-progress: false queue: max + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} outputs: incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} noop_message: ${{ steps.noop.outputs.noop_message }} @@ -1053,7 +1218,7 @@ jobs: steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -1062,15 +1227,16 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent + pattern: "{agent,agent-output-fallback}" + merge-multiple: true path: /tmp/gh-aw/ - name: Setup agent output environment variable id: setup-agent-output-env @@ -1078,32 +1244,29 @@ jobs: run: | mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + - name: Download detection artifact + id: download-detection-artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: detection + path: /tmp/gh-aw/threat-detection/ + - name: Download Safe Outputs Items Manifest + id: download-safe-outputs-manifest + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: safe-outputs-items + merge-multiple: true + path: /tmp/gh-aw/ - name: Collect usage artifact files if: always() continue-on-error: true - run: | - mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection - echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw_info.json /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do - [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" - done - [ -f /tmp/gh-aw/aw_info.json ] && cp /tmp/gh-aw/aw_info.json /tmp/gh-aw/usage/aw_info.json || true - [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true - [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true - [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true - [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true - [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true - [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true - [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true - [ -f /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true - [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true - [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true - [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl - [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl - mkdir -p /tmp/gh-aw/usage/activity - node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs - find /tmp/gh-aw/usage -type f -print | sort + run: bash "${RUNNER_TEMP}/gh-aw/actions/collect_usage_artifact_files.sh" - name: Upload usage artifact if: always() continue-on-error: true @@ -1113,8 +1276,12 @@ jobs: path: | /tmp/gh-aw/usage/aw_info.json /tmp/gh-aw/usage/aw-info.jsonl + /tmp/gh-aw/usage/agent_usage.json /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/evals.jsonl + /tmp/gh-aw/usage/graders/grader_manifest.json + /tmp/gh-aw/usage/graders/grader_results.json /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl @@ -1124,7 +1291,7 @@ jobs: id: restore-daily-aic-cache-conclusion if: always() continue-on-error: true - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: key: agentic-workflow-usage-issuetriage-${{ github.run_id }} restore-keys: agentic-workflow-usage-issuetriage- @@ -1137,15 +1304,17 @@ jobs: with: github-token: ${{ github.token }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context); - const { main } = require('${{ runner.temp }}/gh-aw/actions/write_daily_aic_usage_cache.cjs'); + const { main } = require(path.join(actionsDir, 'write_daily_aic_usage_cache.cjs')); await main(); - name: Save daily AIC usage cache id: save-daily-aic-cache if: always() continue-on-error: true - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: key: agentic-workflow-usage-issuetriage-${{ github.run_id }} path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl @@ -1177,9 +1346,11 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); + const { main } = require(path.join(actionsDir, 'handle_noop_message.cjs')); await main(); - name: Log detection run id: detection_runs @@ -1194,9 +1365,11 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); + const { main } = require(path.join(actionsDir, 'handle_detection_runs.cjs')); await main(); - name: Record missing tool id: missing_tool @@ -1209,9 +1382,11 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); + const { main } = require(path.join(actionsDir, 'missing_tool.cjs')); await main(); - name: Record incomplete id: report_incomplete @@ -1224,9 +1399,11 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); + const { main } = require(path.join(actionsDir, 'report_incomplete_handler.cjs')); await main(); - name: Handle agent failure id: handle_agent_failure @@ -1239,7 +1416,7 @@ jobs: GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} GH_AW_WORKFLOW_ID: "issue-triage" - GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "0" GH_AW_ENGINE_ID: "copilot" GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} @@ -1252,8 +1429,14 @@ jobs: GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_HTTP_400_RESPONSE_ERROR: ${{ needs.agent.outputs.http_400_response_error }} + GH_AW_MAX_CACHE_MISSES_EXCEEDED: ${{ needs.agent.outputs.max_cache_misses_exceeded }} + GH_AW_MISSING_MODEL_PRICING_ERROR: ${{ needs.agent.outputs.missing_model_pricing_error }} + GH_AW_MISSING_MODEL_PRICING_MODEL_NAME: ${{ needs.agent.outputs.missing_model_pricing_model_name }} + GH_AW_SHELL_EXPANSION_GUARD_REJECTED: ${{ needs.agent.outputs.shell_expansion_guard_rejected }} GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }} GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }} GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }} @@ -1262,25 +1445,48 @@ jobs: GH_AW_FAILURE_REPORT_AS_ISSUE: "true" GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" - GH_AW_TIMEOUT_MINUTES: "20" + GH_AW_TIMEOUT_MINUTES: "${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'handle_agent_failure.cjs')); + await main(); + - name: Report failed jobs + id: report_failed_jobs + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_REPORT_FAILED_JOBS: "true" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); + const { main } = require(path.join(actionsDir, 'report_failed_jobs.cjs')); await main(); detection: needs: - activation - agent - if: > - always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest environment: issue-triage permissions: contents: read + timeout-minutes: 10 + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} outputs: aic: ${{ steps.parse_detection_token_usage.outputs.aic }} detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} @@ -1289,7 +1495,7 @@ jobs: steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -1298,15 +1504,22 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download activation artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw - name: Download agent output artifact id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent + pattern: "{agent,agent-output-fallback}" + merge-multiple: true path: /tmp/gh-aw/ - name: Setup agent output environment variable id: setup-agent-output-env @@ -1314,10 +1527,12 @@ jobs: run: | mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi - name: Checkout repository for patch context if: needs.agent.outputs.has_patch == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # --- Threat Detection --- @@ -1326,7 +1541,7 @@ jobs: rm -rf /tmp/gh-aw/sandbox/firewall/logs rm -rf /tmp/gh-aw/sandbox/firewall/audit - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6 ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96 + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f - name: Check if detection needed id: detection_guard if: always() @@ -1350,21 +1565,7 @@ jobs: - name: Prepare threat detection files if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - mkdir -p /tmp/gh-aw/threat-detection/aw-prompts - rm -f /tmp/gh-aw/agent_usage.json - cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true - if [ ! -s /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt ]; then - echo "::warning::ERR_VALIDATION: Missing or empty detection context prompt at /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt. Ensure the agent artifact includes /tmp/gh-aw/aw-prompts/prompt.txt. Detection will continue with fallback workflow context." - fi - cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true - for f in /tmp/gh-aw/aw-*.patch; do - [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true - done - for f in /tmp/gh-aw/aw-*.bundle; do - [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true - done - echo "Prepared threat detection files:" - ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + bash "${RUNNER_TEMP}/gh-aw/actions/prepare_threat_detection_files.sh" - name: Setup threat detection if: always() && steps.detection_guard.outputs.run_detection == 'true' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -1372,83 +1573,52 @@ jobs: WORKFLOW_NAME: "SqlClient Issue Auto-Triage" WORKFLOW_DESCRIPTION: "No description provided" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + GH_AW_DETECTION_SKIP_PROMPT_SUMMARY: "true" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); + const { main } = require(path.join(actionsDir, 'setup_threat_detection.cjs')); await main(); - name: Ensure threat-detection directory and log if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | mkdir -p /tmp/gh-aw/threat-detection touch /tmp/gh-aw/threat-detection/detection.log - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - package-manager-cache: false + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.12 --rootless - name: Install GitHub Copilot CLI - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.63 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" env: GH_HOST: github.com - - name: Install AWF binary - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.7 - - name: Execute GitHub Copilot CLI + GH_AW_COMPILED_VERSION: v0.88.2 + - name: Install threat-detect binary if: always() && steps.detection_guard.outputs.run_detection == 'true' continue-on-error: true - id: detection_agentic_execution - # Copilot CLI tool arguments (sorted): - timeout-minutes: 20 run: | - set -o pipefail - printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt - trap 'rm -f "$HOME/.copilot/settings.json"' EXIT - mkdir -p "$HOME/.copilot" - printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" - export XDG_CONFIG_HOME="$HOME" - touch /tmp/gh-aw/agent-step-summary.md - GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) - export GH_AW_NODE_BIN - export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" - (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) - GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" - printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.7/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5},\"container\":{\"imageTag\":\"0.27.7,squid=sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96,agent=sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c,api-proxy=sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6,cli-proxy=sha256:4757f198a3fa20f88bdbe70be7ae1a05f127d9c0a9e96a5d6460ef40c08fc83d\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" - cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json - export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" - GH_AW_DOCKER_HOST="" - if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then - GH_AW_DOCKER_HOST="${DOCKER_HOST}" - fi - GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" - if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then - GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" - _GH_AW_CHROOT_JSON=$(jq -c --arg src /tmp/gh-aw --arg user "$(id -un)" --argjson uid "$(id -u)" --argjson gid "$(id -g)" --arg home /tmp/gh-aw/home '.chroot={"binariesSourcePath":$src,"identity":{"user":$user,"uid":$uid,"gid":$gid,"home":$home}}' "${RUNNER_TEMP}/gh-aw/awf-config.json") || { echo "chroot config patch failed" >&2; exit 1; } - printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json" - printf '%s\n' "$_GH_AW_CHROOT_JSON" > "/tmp/gh-aw/awf-config.json" - fi - GH_AW_TOOL_CACHE_MOUNT="" - GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" - if [ -d "$GH_AW_TOOL_CACHE" ]; then - if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then - GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" - fi - fi - # shellcheck disable=SC1003,SC2086 - sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ - -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + bash "${RUNNER_TEMP}/gh-aw/actions/install_threat_detect_binary.sh" v0.5.1 + - name: Execute threat detection with AWF + id: detection_agentic_execution + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + timeout-minutes: 10 env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + COPILOT_MODEL: auto + GH_AW_HARNESS_MAX_RETRIES: 0 + GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_TIMEOUT_MINUTES: 20 - GH_AW_VERSION: v0.80.9 + GH_AW_TIMEOUT_MINUTES: 10 + GH_AW_VERSION: v0.88.2 GITHUB_API_URL: ${{ github.api_url }} GITHUB_AW: true GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows @@ -1463,58 +1633,105 @@ jobs: GIT_COMMITTER_NAME: github-actions[bot] RUNNER_TEMP: ${{ runner.temp }} TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} - - name: Parse threat detection token usage for step summary - id: parse_detection_token_usage - if: always() + WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + WORKFLOW_DESCRIPTION: "No description provided" + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + GH_AW_COPILOT_SRC="$(command -v copilot 2>/dev/null || true)" + if [ -z "$GH_AW_COPILOT_SRC" ] || [ ! -x "$GH_AW_COPILOT_SRC" ]; then + echo "GitHub Copilot CLI executable not found on PATH after installation" >&2 + exit 127 + fi + GH_AW_COPILOT_BIN="${RUNNER_TEMP}/gh-aw/bin/copilot" + mkdir -p "${RUNNER_TEMP}/gh-aw/bin" + if [ "$GH_AW_COPILOT_SRC" != "$GH_AW_COPILOT_BIN" ]; then + cp "$GH_AW_COPILOT_SRC" "$GH_AW_COPILOT_BIN" + fi + chmod 755 "$GH_AW_COPILOT_BIN" + + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + if [[ ! "$GH_AW_MAX_AI_CREDITS" =~ ^[0-9]+$ ]]; then + GH_AW_MAX_AI_CREDITS="400" + fi + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.28.12/awf-config.schema.json\",\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-3.7-flash\":[\"copilot/gemini-3.7*flash*\",\"google/gemini-3.7*flash*\",\"gemini/gemini-3.7*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.28.12,squid=sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f,agent=sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202,api-proxy=sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32,cli-proxy=sha256:5250629d48eaedfedf2e948785228e8da29eec2a83cbab58ea0751c14a7b021d\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + _GH_AW_CHROOT_JSON=$(jq -c --arg src "${RUNNER_TEMP}/gh-aw" --arg user "$(id -un)" --argjson uid "$(id -u)" --argjson gid "$(id -g)" --arg home "${RUNNER_TEMP}/gh-aw/home" '.chroot={"binariesSourcePath":$src,"identity":{"user":$user,"uid":$uid,"gid":$gid,"home":$home}}' "${RUNNER_TEMP}/gh-aw/awf-config.json") || { echo "chroot config patch failed" >&2; exit 1; } + printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --mount /tmp/gh-aw:/tmp/gh-aw:rw --mount /tmp/gh-aw/threat-detection:/tmp/gh-aw/threat-detection:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && threat-detect --engine copilot --output /tmp/gh-aw/threat-detection/detection_result.json /tmp/gh-aw/threat-detection' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + - name: Render detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + const { main } = require(path.join(actionsDir, 'render_detection_log.cjs')); await main(); - - name: Upload threat detection log + - name: Copy detection firewall logs + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + run: | + mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall + if [ -d /tmp/gh-aw/sandbox/firewall/logs ]; then mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall/logs && cp -r /tmp/gh-aw/sandbox/firewall/logs/. /tmp/gh-aw/threat-detection/sandbox/firewall/logs/; fi + if [ -d /tmp/gh-aw/sandbox/firewall/audit ]; then mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall/audit && cp -r /tmp/gh-aw/sandbox/firewall/audit/. /tmp/gh-aw/threat-detection/sandbox/firewall/audit/; fi + - name: Upload threat detection artifact if: always() && steps.detection_guard.outputs.run_detection == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: detection - path: /tmp/gh-aw/threat-detection/detection.log + path: | + /tmp/gh-aw/threat-detection/detection_result.json + /tmp/gh-aw/threat-detection/sandbox/firewall/logs/ + /tmp/gh-aw/threat-detection/sandbox/firewall/audit/ if-no-files-found: ignore - - name: Parse and conclude threat detection - id: detection_conclusion + - name: Parse threat detection token usage for step summary + id: parse_detection_token_usage if: always() continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_token_usage.cjs')); + await main(); + - name: Conclude threat detection + id: detection_conclusion + if: always() + continue-on-error: true env: RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" - with: - script: | - try { - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - } catch (loadErr) { - const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; - const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; - const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); - core.error(msg); - core.setOutput('reason', 'parse_error'); - if (continueOnError && !detectionExecutionFailed) { - core.warning('\u26A0\uFE0F ' + msg); - core.setOutput('conclusion', 'warning'); - core.setOutput('success', 'false'); - } else { - core.setOutput('conclusion', 'failure'); - core.setOutput('success', 'false'); - core.setFailed(msg); - } - } + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/conclude_threat_detection.sh" /tmp/gh-aw/threat-detection/detection_result.json safe_outputs: needs: @@ -1525,7 +1742,6 @@ jobs: runs-on: ubuntu-slim environment: issue-triage permissions: - contents: read issues: write pull-requests: write timeout-minutes: 45 @@ -1538,8 +1754,8 @@ jobs: GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" - GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} - GH_AW_ENGINE_VERSION: "1.0.63" + GH_AW_ENGINE_MODEL: "auto" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} GH_AW_WORKFLOW_ID: "issue-triage" GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" @@ -1551,12 +1767,20 @@ jobs: comment_url: ${{ steps.process_safe_outputs.outputs.comment_url }} create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_items_applied: ${{ steps.process_safe_outputs.outputs.items_applied }} + process_safe_outputs_items_cancelled: ${{ steps.process_safe_outputs.outputs.items_cancelled }} + process_safe_outputs_items_deferred: ${{ steps.process_safe_outputs.outputs.items_deferred }} + process_safe_outputs_items_failed: ${{ steps.process_safe_outputs.outputs.items_failed }} + process_safe_outputs_items_skipped: ${{ steps.process_safe_outputs.outputs.items_skipped }} + process_safe_outputs_items_succeeded: ${{ steps.process_safe_outputs.outputs.items_succeeded }} + process_safe_outputs_items_warnings: ${{ steps.process_safe_outputs.outputs.items_warnings }} process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_status: ${{ steps.process_safe_outputs.outputs.status }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -1565,15 +1789,18 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" - name: Download agent output artifact id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent + pattern: "{agent,agent-output-fallback}" + merge-multiple: true path: /tmp/gh-aw/ - name: Setup agent output environment variable id: setup-agent-output-env @@ -1581,7 +1808,9 @@ jobs: run: | mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi - name: Configure GH_HOST for enterprise compatibility id: ghes-host-config shell: bash @@ -1597,16 +1826,18 @@ jobs: env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"hide_older_comments\":true,\"max\":1},\"add_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"remove_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); + const { main } = require(path.join(actionsDir, 'process_safe_outputs.cjs')); await main(); - name: Upload Safe Outputs Items if: always() @@ -1616,5 +1847,5 @@ jobs: path: | /tmp/gh-aw/safe-output-items.jsonl /tmp/gh-aw/temporary-id-map.json + /tmp/gh-aw/safe-output-errors.json if-no-files-found: ignore - diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md index ef157d7f62..100d8baf54 100644 --- a/.github/workflows/issue-triage.md +++ b/.github/workflows/issue-triage.md @@ -41,6 +41,7 @@ if: | )) engine: copilot +model: auto environment: issue-triage @@ -50,6 +51,7 @@ permissions: pull-requests: read tools: + bash: [cat, find, grep] github: min-integrity: none diff --git a/.github/workflows/verify-aw-lock.yml b/.github/workflows/verify-aw-lock.yml index c020335650..eb3cc2e608 100644 --- a/.github/workflows/verify-aw-lock.yml +++ b/.github/workflows/verify-aw-lock.yml @@ -16,9 +16,9 @@ jobs: - uses: actions/checkout@v6 - name: Install gh-aw extension - uses: github/gh-aw-actions/setup-cli@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup-cli@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: - version: v0.80.9 + version: v0.88.2 - name: Recompile agentic workflows run: gh aw compile From 95eda3c9922d00670b22c360d36a0347887a01e0 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:01:18 -0300 Subject: [PATCH 35/51] Pipelines | Read the general-purpose pool name from one shared variable group (#4627) * Read the general-purpose pool name from one shared variable group The pool name was configured three different ways: the CI pipelines read 'ci_var_defaultPoolName' from 'ADO Build properties', the PR pipeline read 'PoolNameDefault' from 'sqlclient-testconfig-v1', and the package pipeline hardcoded ADO-1ES-Pool or ADO-CI-1ES-Pool based on which project it ran in. Three sources for one value, and the package pipeline's copy had to be edited by hand whenever the pool changed. Add a 'sqlclient_pipeline_config' variable group to both the ADO.Net and Public projects, holding 'general_purpose_pool_name' set to that project's pool, and read it at every pipeline root. The name says general-purpose deliberately: we also use special-purpose pools for Always Encrypted, ARM64, Kerberos and Managed Instance jobs, which are still named directly and could get their own variables later. This also removes the project-name check that chose the package pipeline's pool, so 'isInternalBuild' now serves only the signing key argument. No existing variable group is modified. 'ci_var_defaultPoolName' and 'PoolNameDefault' keep their current values; they are simply no longer read. * Rename the variable group to sqlclient-pipeline-config-v1 Match the naming convention used by the other groups in these projects, such as sqlclient-testconfig-v1 and symbols-variables-v3: dashes rather than underscores, and a version suffix. The group was renamed in place in both the ADO.Net and Public projects, so its contents and its pipeline authorization are unchanged. * Keep the pool variable group at pipeline root scope Addresses review feedback on PR #4627. ci-build-nugets-job.yml imported ci-build-variables.yml at job scope, which meant the sqlclient-pipeline-config-v1 group was loaded below the pipeline root. The import was redundant: that job is only reachable from dotnet-sqlclient-ci-core.yml, which already imports the same template at its root, so localFeedPath and packagePath were already in scope. It was also the only job template in the repo importing a shared variables file. Remove it so ci-build-variables.yml is imported at pipeline root only. Also reword the defaultPoolName parameter comment to "most CI jobs", since the Always Encrypted, ARM64, and macOS jobs use special-purpose pools. --- .../ci/package/sqlclient-ci-package-pipeline.yml | 8 ++++---- .../ci/stress/sqlclient-ci-stress-pipeline.yml | 4 ++-- .../common/templates/jobs/ci-build-nugets-job.yml | 3 --- eng/pipelines/dotnet-sqlclient-ci-core.yml | 14 +++++++------- eng/pipelines/github-sync-pipeline.yml | 2 +- eng/pipelines/libraries/ci-build-variables.yml | 1 + eng/pipelines/pr/sqlclient-pr-pipeline.yml | 13 ++++++------- eng/pipelines/pr/variables/pr-variables.yml | 3 +++ 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml b/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml index 8526379bad..41cb134047 100644 --- a/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml +++ b/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml @@ -79,6 +79,9 @@ parameters: - diagnostic variables: + # Provides 'general_purpose_pool_name', the 1ES pool that hosts most jobs in this project. + - group: sqlclient-pipeline-config-v1 + # Whether this is an internal (ADO.Net project) or public (Public project) build. - name: isInternalBuild value: ${{ eq(variables['System.TeamProject'], 'ADO.Net') }} @@ -96,10 +99,7 @@ jobs: displayName: Build NuGet Packages pool: - ${{ if eq(variables.isInternalBuild, true) }}: - name: ADO-1ES-Pool - ${{ else }}: - name: ADO-CI-1ES-Pool + name: $(general_purpose_pool_name) demands: - ImageOverride -equals ${{ parameters.poolImage }} diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml index 1e91534728..813faa09f9 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml @@ -99,13 +99,13 @@ stages: - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self parameters: debug: ${{ parameters.debug }} - poolName: $(ci_var_defaultPoolName) + poolName: $(general_purpose_pool_name) poolImage: ADO-UB24 # Run the stress tests. - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml@self parameters: - poolName: $(ci_var_defaultPoolName) + poolName: $(general_purpose_pool_name) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} warnOnTestFailure: ${{ parameters.warnOnTestFailure }} diff --git a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml index ce8c58b743..9b7a88e6cc 100644 --- a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml @@ -87,9 +87,6 @@ jobs: demands: - imageOverride -equals ${{ parameters.poolImage }} - variables: - - template: /eng/pipelines/libraries/ci-build-variables.yml@self - steps: - ${{ if eq(parameters.debug, true)}}: - powershell: | diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index 3102b9f8a2..8910ccac33 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -94,21 +94,21 @@ parameters: - Debug - Release - # The name of the 1ES pool that all CI jobs run in. + # The name of the general-purpose 1ES pool that most CI jobs run in. # # This is the single place in the CI pipelines where the pool name is read # from a variable group; every stage and job below receives it as a # parameter so that the value flows down from here. # - # 'ci_var_defaultPoolName' is defined in the 'ADO Build properties' variable - # group (see /eng/pipelines/libraries/ci-build-variables.yml), for both the - # Public and ADO.Net projects. The PR pipelines are configured from a - # different variable group and use '$(PoolNameDefault)' instead; see - # /eng/pipelines/pr/sqlclient-pr-pipeline.yml. + # 'general_purpose_pool_name' is defined in the 'sqlclient-pipeline-config-v1' + # variable group (see /eng/pipelines/libraries/ci-build-variables.yml), for + # both the Public and ADO.Net projects, and names each project's own pool. + # Special-purpose pools, such as the Always Encrypted and ARM64 pools used + # below, are still named directly. # - name: defaultPoolName type: string - default: $(ci_var_defaultPoolName) + default: $(general_purpose_pool_name) # The timeout, in minutes, for each test job. - name: testJobTimeout diff --git a/eng/pipelines/github-sync-pipeline.yml b/eng/pipelines/github-sync-pipeline.yml index 6d186a10b1..eb8c3081c6 100644 --- a/eng/pipelines/github-sync-pipeline.yml +++ b/eng/pipelines/github-sync-pipeline.yml @@ -64,7 +64,7 @@ jobs: - job: SyncGitHub displayName: Sync GitHub to ADO pool: - name: $(ci_var_defaultPoolName) + name: $(general_purpose_pool_name) demands: - imageOverride -equals ADO-UB24 diff --git a/eng/pipelines/libraries/ci-build-variables.yml b/eng/pipelines/libraries/ci-build-variables.yml index 5486fa9da5..774be86113 100644 --- a/eng/pipelines/libraries/ci-build-variables.yml +++ b/eng/pipelines/libraries/ci-build-variables.yml @@ -9,6 +9,7 @@ # The buildSuffix itself is set by each pipeline via the core template parameter. variables: + - group: sqlclient-pipeline-config-v1 - group: ADO Build properties - group: ADO Test Configuration Properties diff --git a/eng/pipelines/pr/sqlclient-pr-pipeline.yml b/eng/pipelines/pr/sqlclient-pr-pipeline.yml index 640c448c13..eab08243e8 100644 --- a/eng/pipelines/pr/sqlclient-pr-pipeline.yml +++ b/eng/pipelines/pr/sqlclient-pr-pipeline.yml @@ -103,15 +103,14 @@ variables: - template: /eng/pipelines/pr/variables/pr-variables.yml@self stages: - # NOTE: '$(PoolNameDefault)' comes from the 'sqlclient-testconfig-v1' + # NOTE: '$(general_purpose_pool_name)' comes from the 'sqlclient-pipeline-config-v1' # variable group, which is imported by # /eng/pipelines/pr/variables/pr-variables.yml (included above). It is # referenced here, at the pipeline root, and passed down to every stage as a # parameter, so that no template reads the pool name from a variable group # directly. # - # The CI pipelines are configured from a different variable group and use - # '$(ci_var_defaultPoolName)' instead; see + # The same variable group and variable are used by the CI pipelines; see # /eng/pipelines/dotnet-sqlclient-ci-core.yml. # Stage 1a: Build and pack all projects in the repository @@ -121,14 +120,14 @@ stages: buildSuffix: pr stageName: ${{ variables.stageNamePack }} packArtifactBaseName: ${{ variables.packArtifactBaseName }} - poolName: $(PoolNameDefault) + poolName: $(general_purpose_pool_name) poolImage: ADO-UB24 # Stage 1b: Generate secrets - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml@self parameters: stageName: ${{ variables.stageNameSecrets }} - poolName: $(PoolNameDefault) + poolName: $(general_purpose_pool_name) poolImage: ADO-UB24 # Stage 2: Execute tests and collect code coverage @@ -137,7 +136,7 @@ stages: buildConfiguration: Debug buildSuffix: pr dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - poolName: $(PoolNameDefault) + poolName: $(general_purpose_pool_name) stageNamePack: ${{ variables.stageNamePack }} stageNameSecrets: ${{ variables.stageNameSecrets }} testResultsArtifactBaseName: ${{ variables.testResultsArtifactBaseName }} @@ -159,7 +158,7 @@ stages: - template: /eng/pipelines/pr/stages/collect-coverage-stage.yml@self parameters: coverageArtifactBaseName: ${{ variables.coverageArtifactBaseName }} - poolName: $(PoolNameDefault) + poolName: $(general_purpose_pool_name) poolImage: ADO-UB24 dependsOn: - ${{ each platform in parameters.platforms }}: diff --git a/eng/pipelines/pr/variables/pr-variables.yml b/eng/pipelines/pr/variables/pr-variables.yml index 5d856c4859..3e97ccb8ac 100644 --- a/eng/pipelines/pr/variables/pr-variables.yml +++ b/eng/pipelines/pr/variables/pr-variables.yml @@ -23,6 +23,9 @@ variables: # UserManagedIdentityClientId - group: sqlclient-testconfig-v1 + # Provides 'general_purpose_pool_name', the 1ES pool that hosts most jobs in this project. + - group: sqlclient-pipeline-config-v1 + # General Variables ====================================================== # Name to use for the packaging stage. This is to ensure that the packaging stage and the test From a4b8485bd4d9ac75a0463aea8982fb31ebe732c4 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:02:33 -0700 Subject: [PATCH 36/51] Validate localized SqlClient resources in OneBranch builds (#4635) * Add localization validation to OneBranch pipelines Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Run localization validation in SqlClient build Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Allow approved localization value matches Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b28f059a-cfb7-4fa1-95b7-0947e3c965a1 * Harden localization validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b28f059a-cfb7-4fa1-95b7-0947e3c965a1 * Handle localization handback delay Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b28f059a-cfb7-4fa1-95b7-0947e3c965a1 * Honor warning-only allowlist validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b28f059a-cfb7-4fa1-95b7-0947e3c965a1 * Always enforce localization validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b28f059a-cfb7-4fa1-95b7-0947e3c965a1 * Clarify localization validation docs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b28f059a-cfb7-4fa1-95b7-0947e3c965a1 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b28f059a-cfb7-4fa1-95b7-0947e3c965a1 --- .config/LocalizationValidationAllowlist.json | 110 ++++++++++ .../onebranch-pipeline-design.instructions.md | 6 +- .../onebranch/jobs/build-buildproj-job.yml | 5 + .../onebranch/scripts/tests/README.md | 1 + .../tests/validate-localization.Tests.ps1 | 190 ++++++++++++++++++ .../scripts/validate-localization.ps1 | 180 +++++++++++++++++ .../steps/validate-localization-step.yml | 16 ++ 7 files changed, 507 insertions(+), 1 deletion(-) create mode 100644 .config/LocalizationValidationAllowlist.json create mode 100644 eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 create mode 100644 eng/pipelines/onebranch/scripts/validate-localization.ps1 create mode 100644 eng/pipelines/onebranch/steps/validate-localization-step.yml diff --git a/.config/LocalizationValidationAllowlist.json b/.config/LocalizationValidationAllowlist.json new file mode 100644 index 0000000000..0a3cbfea6b --- /dev/null +++ b/.config/LocalizationValidationAllowlist.json @@ -0,0 +1,110 @@ +{ + "_comment": [ + "These culture/key pairs intentionally match the English source text.", + "Each pair was verified in the internal LCL source as localized (Stat=Loc, Orig=New). Remove an entry when its localized value changes." + ], + "AllowedEnglishValueMatches": { + "Strings.cs.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_Data", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.de.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId" + ], + "Strings.es.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass" + ], + "Strings.fr.resx": [ + "DataCategory_InfoMessage", + "DataCategory_Notification", + "DataCategory_Source", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SqlMisc_NullString" + ], + "Strings.it.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.ja.resx": [ + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId" + ], + "Strings.ko.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.pl.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.pt-BR.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass" + ], + "Strings.ru.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExOriginalClientConnectionId" + ], + "Strings.tr.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.zh-Hans.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.zh-Hant.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ] + } +} diff --git a/.github/instructions/onebranch-pipeline-design.instructions.md b/.github/instructions/onebranch-pipeline-design.instructions.md index 75ea6ea303..7d42d279e7 100644 --- a/.github/instructions/onebranch-pipeline-design.instructions.md +++ b/.github/instructions/onebranch-pipeline-design.instructions.md @@ -26,9 +26,13 @@ Respect this graph when modifying build stages: 5. `Microsoft.Data.SqlClient.Extensions.Azure` — depends on Abstractions + Logging 6. `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` — depends on SqlClient + Abstractions + Logging +## Localization Validation + +The SqlClient build job runs `steps/validate-localization-step.yml` before building the driver. Validation always fails the build for missing or obsolete keys, empty localized values whose English value is non-empty, and untranslated resources. Approved identical translations are listed by culture and resource key in `.config/LocalizationValidationAllowlist.json`. + ## Build Stages -Defined in `stages/build-stages.yml`. Four build stages plus validation, ordered by dependency: +Defined in `stages/build-stages.yml`. Four build stages plus package validation are ordered by dependency: - **`build_independent`** (Stage 1) — Logging and SqlServer.Server in parallel; no inter-package dependencies - **`build_abstractions`** (Stage 2) — Abstractions; `dependsOn: build_independent`; downloads Logging artifact diff --git a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml index 824776575d..6576627a6a 100644 --- a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml +++ b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml @@ -137,6 +137,11 @@ jobs: steps: - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self + # Localized resources ship with the SqlClient driver. Validate them before analysis and + # building so missing or untranslated strings fail every SqlClient build. + - ${{ if eq(parameters.packageShortName, 'SqlClient') }}: + - template: /eng/pipelines/onebranch/steps/validate-localization-step.yml@self + - ${{ each package in parameters.dependencies }}: # Build the dependency version arguments passed to the build/pack/analysis steps. The # SqlClient family shares a single version via Central Package Management, and those steps diff --git a/eng/pipelines/onebranch/scripts/tests/README.md b/eng/pipelines/onebranch/scripts/tests/README.md index b2ea3754c1..593c7768d6 100644 --- a/eng/pipelines/onebranch/scripts/tests/README.md +++ b/eng/pipelines/onebranch/scripts/tests/README.md @@ -32,6 +32,7 @@ Invoke-Pester ./publish-symbols.Tests.ps1 -Output Detailed | Area | What's tested | | --------------------- | ---------------------------------------------------------------- | | Version computation | Canonical output parsing, effective package selection, target version composition, and failures | +| Localization validation | Missing, obsolete, or empty strings, English-value matches, and culture-specific allowlisting | | Parameter validation | Empty strings rejected for all mandatory parameters | | URL construction | Base URL, register URL, request URL built from parameters | | Request bodies | Registration body, default publish flags, flag overrides | diff --git a/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 new file mode 100644 index 0000000000..5f27b883d3 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 @@ -0,0 +1,190 @@ +<# +.SYNOPSIS + Pester tests for validate-localization.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'validate-localization.ps1' + + function Set-ResourceFile { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][hashtable]$Strings + ) + + $document = [System.Xml.XmlDocument]::new() + $root = $document.CreateElement('root') + $null = $document.AppendChild($root) + foreach ($entry in $Strings.GetEnumerator()) { + $data = $document.CreateElement('data') + $data.SetAttribute('name', $entry.Key) + + $value = $document.CreateElement('value') + $value.InnerText = $entry.Value + $null = $data.AppendChild($value) + $null = $root.AppendChild($data) + } + + $document.Save($Path) + } + + function New-ResourcesDirectory { + $path = Join-Path $TestDrive ([guid]::NewGuid().ToString('n')) + New-Item -ItemType Directory -Path $path | Out-Null + return $path + } +} + +Describe 'validate-localization.ps1' { + It 'accepts complete localized files with translated values' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour'; Farewell = 'Au revoir' } + + { & $scriptPath -ResourcesDirectory $resources } | Should -Not -Throw + } + + It 'fails when a localized file is missing an English key' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.de.resx') @{ Greeting = 'Hallo' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when a localized value matches a non-empty English value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.ja.resx') @{ Greeting = 'Hello'; Unused = '' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when no localized resource files exist' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*No localized Strings.*.resx files were found*' + } + + It 'fails when a non-empty English string has an empty localized value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.es.resx') @{ Greeting = ' ' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'accepts empty localized values when the English value is also empty' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.ko.resx') @{ Unused = '' } + + { & $scriptPath -ResourcesDirectory $resources } | Should -Not -Throw + } + + It 'fails when a resource data element has no value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + [xml]$localized = Get-Content -LiteralPath (Join-Path $resources 'Strings.fr.resx') + $valueNode = $localized.SelectSingleNode('/root/data/value') + $null = $valueNode.ParentNode.RemoveChild($valueNode) + $localized.Save((Join-Path $resources 'Strings.fr.resx')) + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*contains a element without a name or value*' + } + + It 'accepts an approved English-value match from the allowlist' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Not -Throw + } + + It 'does not allowlist a missing localized key' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Farewell') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects allowlist for unknown resource keys' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Unknown') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'scopes approved English-value matches to one localized file' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.de.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } + @{ AllowedEnglishValueMatches = @{ 'Strings.de.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when a localized file contains a key absent from Strings.resx' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour'; Obsolete = 'Ancien' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects an allowlist entry after the localized value is translated' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects allowlist entries for empty English values' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Unused = '' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Unused') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*does not have a non-empty English value*' + } +} diff --git a/eng/pipelines/onebranch/scripts/validate-localization.ps1 b/eng/pipelines/onebranch/scripts/validate-localization.ps1 new file mode 100644 index 0000000000..1922c78c5b --- /dev/null +++ b/eng/pipelines/onebranch/scripts/validate-localization.ps1 @@ -0,0 +1,180 @@ +<# +.SYNOPSIS + Validates localized Strings.*.resx files against Strings.resx. + +.PARAMETER ResourcesDirectory + Directory containing the English and localized Strings.resx files. + +.PARAMETER AllowlistPath + Optional JSON file containing approved English-value matches grouped by localized filename. +#> + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# See the LICENSE file in the project root for more information. + +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ResourcesDirectory, + + [string]$AllowlistPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Get-ResourceStrings { + param([Parameter(Mandatory)][string]$Path) + + $document = [System.Xml.Linq.XDocument]::Load($Path) + $strings = [System.Collections.Generic.Dictionary[string, string]]::new([System.StringComparer]::Ordinal) + foreach ($data in $document.Root.Elements('data')) { + $name = $data.Attribute('name') + $value = $data.Element('value') + if ($null -eq $name -or $null -eq $value) { + throw "Resource file '$Path' contains a element without a name or value." + } + + if ($strings.ContainsKey($name.Value)) { + throw "Resource file '$Path' contains duplicate key '$($name.Value)'." + } + + $strings.Add($name.Value, $value.Value) + } + + return $strings +} + +# Discover the neutral English resource and every culture-specific resource. The English file is +# the source of truth for the required key set and value-comparison checks below. +$resourcesPath = (Resolve-Path -LiteralPath $ResourcesDirectory).Path +$englishPath = Join-Path $resourcesPath 'Strings.resx' +if (-not (Test-Path -LiteralPath $englishPath -PathType Leaf)) { + throw "English resource file '$englishPath' was not found." +} + +$localizedFiles = @(Get-ChildItem -LiteralPath $resourcesPath -Filter 'Strings.*.resx' -File | Sort-Object Name) +if ($localizedFiles.Count -eq 0) { + throw "No localized Strings.*.resx files were found in '$resourcesPath'." +} + +$englishStrings = Get-ResourceStrings -Path $englishPath +$localizedFilesByName = [System.Collections.Generic.Dictionary[string, System.IO.FileInfo]]::new([System.StringComparer]::Ordinal) +foreach ($localizedFile in $localizedFiles) { + $localizedFilesByName.Add($localizedFile.Name, $localizedFile) +} + +# Load culture/key-specific exceptions for translations intentionally identical to English. The +# configuration is validated against the current resources so stale filenames and keys fail the +# build instead of silently weakening future validation. +$failures = [System.Collections.Generic.List[string]]::new() +$allowedEnglishMatches = [System.Collections.Generic.Dictionary[string, System.Collections.Generic.HashSet[string]]]::new([System.StringComparer]::Ordinal) +if (-not [string]::IsNullOrWhiteSpace($AllowlistPath)) { + if (-not (Test-Path -LiteralPath $AllowlistPath -PathType Leaf)) { + throw "Localization allowlist file '$AllowlistPath' was not found." + } + + $configuration = Get-Content -LiteralPath $AllowlistPath -Raw | ConvertFrom-Json + $englishValueMatchesProperty = $configuration.PSObject.Properties['AllowedEnglishValueMatches'] + if ($null -eq $englishValueMatchesProperty) { + throw "Localization allowlist file '$AllowlistPath' must define 'AllowedEnglishValueMatches'." + } + + foreach ($fileProperty in $englishValueMatchesProperty.Value.PSObject.Properties) { + if (-not $localizedFilesByName.ContainsKey($fileProperty.Name)) { + $failures.Add("Localization allowlist file references unknown resource file '$($fileProperty.Name)'.") + continue + } + + $keys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($key in @($fileProperty.Value)) { + if ([string]::IsNullOrWhiteSpace($key)) { + throw "Localization allowlist file contains an empty resource key for '$($fileProperty.Name)'." + } + if (-not $englishStrings.ContainsKey($key)) { + $failures.Add("Localization allowlist file references unknown resource key '$key' for '$($fileProperty.Name)'.") + continue + } + if ([string]::IsNullOrEmpty($englishStrings[$key])) { + throw "Localization allowlist key '$key' for '$($fileProperty.Name)' does not have a non-empty English value." + } + if (-not $keys.Add($key)) { + throw "Localization allowlist file contains duplicate key '$key' for '$($fileProperty.Name)'." + } + } + $allowedEnglishMatches.Add($fileProperty.Name, $keys) + } +} + +$allowedMatchCount = 0 +foreach ($localizedFile in $localizedFiles) { + $localizedStrings = Get-ResourceStrings -Path $localizedFile.FullName + + # Validate the key sets in both directions, then compare every non-empty English value with + # its localized value. Empty neutral values may remain empty because they contain no text to + # translate. + $missingOrEmptyKeys = [System.Collections.Generic.List[string]]::new() + $localizedOnlyKeys = [System.Collections.Generic.List[string]]::new() + $englishMatches = [System.Collections.Generic.List[string]]::new() + $staleAllowlistKeys = [System.Collections.Generic.List[string]]::new() + + foreach ($localizedKey in $localizedStrings.Keys) { + if (-not $englishStrings.ContainsKey($localizedKey)) { + $localizedOnlyKeys.Add($localizedKey) + } + } + + foreach ($entry in $englishStrings.GetEnumerator()) { + if (-not $localizedStrings.ContainsKey($entry.Key) -or + (-not [string]::IsNullOrEmpty($entry.Value) -and + [string]::IsNullOrWhiteSpace($localizedStrings[$entry.Key]))) { + $missingOrEmptyKeys.Add($entry.Key) + } + elseif (-not [string]::IsNullOrEmpty($entry.Value) -and + [System.StringComparer]::Ordinal.Equals($entry.Value, $localizedStrings[$entry.Key])) { + if ($allowedEnglishMatches.ContainsKey($localizedFile.Name) -and + $allowedEnglishMatches[$localizedFile.Name].Contains($entry.Key)) { + $allowedMatchCount++ + } + else { + $englishMatches.Add($entry.Key) + } + } + # An allowlist entry must be removed after its localized value changes; otherwise a future + # regression to the English value could be hidden by an obsolete exception. + elseif ($allowedEnglishMatches.ContainsKey($localizedFile.Name) -and + $allowedEnglishMatches[$localizedFile.Name].Contains($entry.Key)) { + $staleAllowlistKeys.Add($entry.Key) + } + } + + if ($missingOrEmptyKeys.Count -gt 0) { + $missingOrEmptyKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): missing keys or empty values: $($missingOrEmptyKeys -join ', ')") + } + if ($localizedOnlyKeys.Count -gt 0) { + $localizedOnlyKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): keys not found in Strings.resx: $($localizedOnlyKeys -join ', ')") + } + if ($englishMatches.Count -gt 0) { + $englishMatches.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): untranslated values match Strings.resx: $($englishMatches -join ', ')") + } + if ($staleAllowlistKeys.Count -gt 0) { + $staleAllowlistKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): allowlist entries no longer match Strings.resx: $($staleAllowlistKeys -join ', ')") + } +} + +if ($failures.Count -gt 0) { + foreach ($failure in $failures) { + Write-Host "##vso[task.logissue type=error]$failure" + } + $errorNoun = if ($failures.Count -eq 1) { 'error' } else { 'errors' } + throw "Localization validation failed with $($failures.Count) $errorNoun. Review the preceding errors." +} + +$fileNoun = if ($localizedFiles.Count -eq 1) { 'file' } else { 'files' } +Write-Host "Localization validation passed for $($localizedFiles.Count) localized $fileNoun. Resource keys checked: $($englishStrings.Count); approved English-value matches allowlisted: $allowedMatchCount." diff --git a/eng/pipelines/onebranch/steps/validate-localization-step.yml b/eng/pipelines/onebranch/steps/validate-localization-step.yml new file mode 100644 index 0000000000..8cb590f6d0 --- /dev/null +++ b/eng/pipelines/onebranch/steps/validate-localization-step.yml @@ -0,0 +1,16 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +steps: + - task: PowerShell@2 + displayName: 'Validate localized resources' + inputs: + targetType: filePath + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/pipelines/onebranch/scripts/validate-localization.ps1 + arguments: >- + -ResourcesDirectory "$(Build.SourcesDirectory)/src/Microsoft.Data.SqlClient/src/Resources" + -AllowlistPath "$(Build.SourcesDirectory)/.config/LocalizationValidationAllowlist.json" From 5579c13a990b712e58ac678ed14da2ce17295bd2 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:10:30 -0300 Subject: [PATCH 37/51] Capture real diagnostics and retry the SQL Server container on macOS CI (#4667) --- .../steps/configure-sql-server-macos-step.yml | 172 ++++++++++++------ .../steps/publish-test-results-step.yml | 15 +- 2 files changed, 130 insertions(+), 57 deletions(-) diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml index 772433ba3a..0ffeb4f2ab 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml @@ -141,84 +141,143 @@ steps: # Password for the SA user (required) MSSQL_SA_PW="${{ parameters.saPassword }}" - # Start the container and fail fast (with diagnostics) if it does not launch, - # rather than silently falling through to the connection-wait loop. - if ! docker run --platform linux/amd64 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2025-latest; then - echo "ERROR: Failed to start the sql1 container." - docker ps -a --filter "name=^/sql1$" - exit 1 - fi + # Collect everything needed to diagnose a container that crashed or never + # became ready. + # + # The mssql dump collector (paldumper) writes hundreds of + # "find: '/proc/N/...': Permission denied" lines to the container's stderr + # whenever sqlservr faults. That noise used to fill the entire captured + # window and hide the actual SQL Server error, so it is filtered out here + # and both the head and the tail of the log are printed. + dumpSqlDiagnostics() + { + set +x + + echo "--- Container status ---" + docker ps -a --filter "name=^/sql1$" || true - sleep 10 + echo "--- Container state ---" + docker inspect sql1 --format 'ExitCode={{.State.ExitCode}} OOMKilled={{.State.OOMKilled}} Error="{{.State.Error}}" StartedAt={{.State.StartedAt}} FinishedAt={{.State.FinishedAt}}' || true - docker ps -a + echo "--- SQL Server errorlog (last 80 lines) ---" + # docker cp works against a stopped container, so this survives a crash. + if docker cp sql1:/var/opt/mssql/log/errorlog "$SQL_ERRORLOG" 2>/dev/null; then + tail -80 "$SQL_ERRORLOG" + else + echo "(errorlog not available)" + fi - # Connect to the SQL Server container and get its version. - # - # With Rosetta 2 emulation, SQL Server starts much faster than under full - # QEMU emulation, but it can still take a minute or two. We allow up to - # 6 minutes (72 attempts × 5 seconds) as a generous upper bound. + echo "--- Container logs, dump-collector noise filtered (first 60 lines) ---" + docker logs sql1 2>&1 | grep -vE "^(find|dmesg|timeout): " | head -60 || true + echo "--- Container logs, dump-collector noise filtered (last 40 lines) ---" + docker logs sql1 2>&1 | grep -vE "^(find|dmesg|timeout): " | tail -40 || true - # Wait 5 seconds between attempts. - delay=5 + echo "--- sqlcmd errors ---" + cat "$SQLCMD_ERRORS" 2>/dev/null || echo "(none)" - # Try up to 72 times (~6 minutes) to connect. - maxAttempts=72 + echo "--- Host capacity ---" + echo "hw.ncpu=$(sysctl -n hw.ncpu 2>/dev/null) hw.memsize=$(sysctl -n hw.memsize 2>/dev/null)" + vm_stat || true - # Attempt counter. - attempt=1 + echo "--- Guest capacity ---" + colima ssh -- free -m || true - # Flag to indicate when SQL Server is ready to accept connections. - ready=0 + set -x + } - while [ $attempt -le $maxAttempts ] + SQL_ERRORLOG=$(Agent.TempDirectory)/mssql_errorlog + + # Start SQL Server, retrying the whole container lifecycle. sqlservr + # intermittently core dumps within a minute or two of starting on these + # agents (the container goes to "Exited (1)" while the readiness loop is + # still polling), which previously failed the entire step on the first + # occurrence. Colima startup and the image pull already retry; the + # container did not. + runAttempts=3 + sqlReady=0 + + for ((r=1; r<=runAttempts; r++)) do + echo "Starting SQL Server container (attempt #$r of $runAttempts)..." + + # Clear any container and sqlcmd output left behind by a prior attempt. + docker rm -f sql1 >/dev/null 2>&1 || true + : > "$SQLCMD_ERRORS" + + if ! docker run --platform linux/amd64 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2025-latest; then + echo "ERROR: Failed to start the sql1 container (attempt #$r of $runAttempts)." + dumpSqlDiagnostics + continue + fi + + sleep 10 + + docker ps -a - echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." + # Connect to the SQL Server container and get its version. + # + # With Rosetta 2 emulation, SQL Server starts much faster than under full + # QEMU emulation, but it can still take a minute or two. We allow up to + # 6 minutes (72 attempts × 5 seconds) as a generous upper bound. - # -C trusts the self-signed certificate inside the container. - sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" >> $SQLCMD_ERRORS 2>&1 + # Wait 5 seconds between attempts. + delay=5 - # If the command was successful, then the SQL Server is ready. - if [ $? -eq 0 ]; then - ready=1 + # Try up to 72 times (~6 minutes) to connect. + maxAttempts=72 + + # Attempt counter. + attempt=1 + + # Set when the container died before SQL Server accepted connections. + crashed=0 + + while [ $attempt -le $maxAttempts ] + do + + echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." + + # -C trusts the self-signed certificate inside the container. + if sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" >> "$SQLCMD_ERRORS" 2>&1; then + sqlReady=1 + break + fi + + # Verify the container is still running; no point retrying if it crashed. + if ! docker ps --filter "name=^/sql1$" --filter "status=running" --format '{{.Names}}' | grep -Fxq 'sql1'; then + echo "ERROR: sql1 container is no longer running (attempt #$r of $runAttempts)." + crashed=1 + break + fi + + # Increment the attempt counter. + ((attempt++)) + + # Wait before trying again. + sleep $delay + + done + + if [ $sqlReady -eq 1 ]; then break fi - # Verify the container is still running; no point retrying if it crashed. - if ! docker ps --filter "name=^/sql1$" --filter "status=running" --format '{{.Names}}' | grep -Fxq 'sql1'; then - echo "ERROR: sql1 container is no longer running." - docker ps -a --filter "name=^/sql1$" - echo "--- Container logs ---" - docker logs sql1 2>&1 | tail -50 - rm -f $SQLCMD_ERRORS - exit 1 + if [ $crashed -ne 1 ]; then + echo "ERROR: Cannot connect to SQL Server after $maxAttempts attempts (attempt #$r of $runAttempts)." fi - # Increment the attempt counter. - ((attempt++)) - - # Wait before trying again. - sleep $delay - + dumpSqlDiagnostics done # Is the SQL Server ready? - if [ $ready -eq 0 ] + if [ $sqlReady -ne 1 ] then - # No, so report the error(s) and exit. - echo "Cannot connect to SQL Server after $maxAttempts attempts; installation aborted." - echo "--- sqlcmd errors ---" - cat $SQLCMD_ERRORS - echo "--- Container status ---" - docker ps -a --filter "name=^/sql1$" - echo "--- Container logs (last 80 lines) ---" - docker logs sql1 2>&1 | tail -80 - rm -f $SQLCMD_ERRORS + echo "ERROR: SQL Server did not become ready after $runAttempts container attempts; installation aborted." + rm -f "$SQLCMD_ERRORS" "$SQL_ERRORLOG" exit 1 fi - rm -f $SQLCMD_ERRORS + rm -f "$SQLCMD_ERRORS" "$SQL_ERRORLOG" echo "Use sqlcmd to show which IP addresses are being listened on..." echo 0.0.0.0 @@ -247,5 +306,6 @@ steps: # boot fails here instead of consuming the whole test job. Measured worst # case is ~24 minutes: Colima boot ~6, the SQL image pull ~10 (7 of which is # extraction inside the VM), and up to 6 more waiting for SQL to accept - # connections. - timeoutInMinutes: 40 + # connections. The container is now started up to 3 times, so the readiness + # wait can cost 18 minutes rather than 6 in the pathological case. + timeoutInMinutes: 55 diff --git a/eng/pipelines/common/templates/steps/publish-test-results-step.yml b/eng/pipelines/common/templates/steps/publish-test-results-step.yml index 9cb356aa45..8e06e9dece 100644 --- a/eng/pipelines/common/templates/steps/publish-test-results-step.yml +++ b/eng/pipelines/common/templates/steps/publish-test-results-step.yml @@ -59,9 +59,22 @@ steps: Get-ChildItem -Filter "*.coverage" -Recurse displayName: '[Debug] List test result coverage files' +# When an earlier step fails (e.g. SQL Server setup), no TestResults directory is +# produced and PublishPipelineArtifact fails with "Path does not exist", adding a +# second error that buries the real one. Gate the publish on the directory +# actually existing. +- pwsh: | + $exists = Test-Path -Path 'TestResults' -PathType Container + if (-not $exists) { + Write-Host 'No TestResults directory was produced; skipping test artifact publish.' + } + Write-Host "##vso[task.setvariable variable=HasTestResults]$($exists.ToString().ToLowerInvariant())" + displayName: 'Check for test results' + condition: succeededOrFailed() + - task: PublishPipelineArtifact@1 displayName: 'Publish Test Artifacts' inputs: targetPath: TestResults artifact: '${{parameters.targetFramework }}WinAz$(System.JobId)' - condition: succeededOrFailed() + condition: and(succeededOrFailed(), eq(variables['HasTestResults'], 'true')) From f2310e93f14c2589718551339b7fc7c33a72ca66 Mon Sep 17 00:00:00 2001 From: Benjamin Russell Date: Thu, 10 Sep 2026 18:09:28 -0500 Subject: [PATCH 38/51] Hotfix Release Notes (#4672) * Release notes for v6.1.7 * Release notes for v7.0.3 --- CHANGELOG.md | 59 +++++++++ release-notes/6.1/6.1.7.md | 103 ++++++++++++++++ release-notes/6.1/README.md | 1 + release-notes/7.0/7.0.3.md | 115 ++++++++++++++++++ release-notes/7.0/README.md | 1 + .../Extensions/Abstractions/7.0/7.0.3.md | 17 +++ .../Extensions/Abstractions/7.0/README.md | 1 + release-notes/Extensions/Azure/7.0/7.0.3.md | 39 ++++++ release-notes/Extensions/Azure/7.0/README.md | 1 + release-notes/Internal/Logging/7.0/7.0.3.md | 20 +++ release-notes/Internal/Logging/7.0/README.md | 1 + .../AzureKeyVaultProvider/7.0/7.0.3.md | 21 ++++ .../AzureKeyVaultProvider/7.0/README.md | 1 + 13 files changed, 380 insertions(+) create mode 100644 release-notes/6.1/6.1.7.md create mode 100644 release-notes/7.0/7.0.3.md create mode 100644 release-notes/Extensions/Abstractions/7.0/7.0.3.md create mode 100644 release-notes/Extensions/Azure/7.0/7.0.3.md create mode 100644 release-notes/Internal/Logging/7.0/7.0.3.md create mode 100644 release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md diff --git a/CHANGELOG.md b/CHANGELOG.md index dd60021553..3901428151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,65 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) > **Note:** Releases are sorted in reverse chronological order (newest first). +## [Stable Release 7.0.3] - 2026-09-10 + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4599](https://github.com/dotnet/SqlClient/pull/4599)) + +### Fixed + +- Fixed a `SqlBulkCopy` regression in environments where the application login cannot read `sys.all_columns`. Bulk copy now falls back to the earlier column-discovery behavior when that permission is unavailable. Support for hidden columns and SQL Graph column aliases still requires access to the metadata view. + ([#4370](https://github.com/dotnet/SqlClient/issues/4370), [#4306](https://github.com/dotnet/SqlClient/pull/4306), [#4402](https://github.com/dotnet/SqlClient/pull/4402)) + +- Fixed a memory-allocation regression in connection and command operations caused by formatting diagnostic strings even when tracing was disabled. Also corrected trace messages that reported an incorrect object ID or could throw `FormatException` when traced values contained braces. + ([#4528](https://github.com/dotnet/SqlClient/pull/4528), [#4533](https://github.com/dotnet/SqlClient/pull/4533)) + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4583](https://github.com/dotnet/SqlClient/pull/4583)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4553](https://github.com/dotnet/SqlClient/pull/4553)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561)) + +- Fixed authentication state handling so clearing `SqlConnection.AccessToken`, `AccessTokenCallback`, or `SspiContextProvider` preserves the other authentication values in the connection pool key. Cloning a connection or updating its credential also preserves its `SspiContextProvider`. Combining a non-null `SspiContextProvider` with `AccessToken` or `AccessTokenCallback` now throws `InvalidOperationException` instead of silently discarding authentication state; applications must use one authentication mechanism at a time. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561), [#4644](https://github.com/dotnet/SqlClient/pull/4644)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4663](https://github.com/dotnet/SqlClient/pull/4663)) + +### Companion packages + +- Released `Microsoft.Data.SqlClient.Extensions.Azure` 7.0.3 with the Entra ID authority parsing fix for Dataverse/Dynamics 365 connections. See [release notes](release-notes/Extensions/Azure/7.0/7.0.3.md). +- Released version-aligned `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, and `Microsoft.Data.SqlClient.Internal.Logging` 7.0.3 with no functional or API changes. See the [Azure Key Vault provider](release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md), [Abstractions](release-notes/Extensions/Abstractions/7.0/7.0.3.md), and [Logging](release-notes/Internal/Logging/7.0/7.0.3.md) release notes. + +## [Stable Release 6.1.7] - 2026-09-10 + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4598](https://github.com/dotnet/SqlClient/pull/4598)) + +### Fixed + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4584](https://github.com/dotnet/SqlClient/pull/4584)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4552](https://github.com/dotnet/SqlClient/pull/4552)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed token authentication state handling so clearing `SqlConnection.AccessToken` preserves an existing `AccessTokenCallback` in the connection pool key, and clearing `AccessTokenCallback` preserves an existing `AccessToken`. Callback-based authentication now also follows the same prelogin server-certificate validation rules as an explicitly supplied access token. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4664](https://github.com/dotnet/SqlClient/pull/4664)) + ## [Preview Release 7.1.0-preview3] - 2026-08-26 This update brings the following changes since the [7.1.0-preview2](release-notes/7.1/7.1.0-preview2.md) release. diff --git a/release-notes/6.1/6.1.7.md b/release-notes/6.1/6.1.7.md new file mode 100644 index 0000000000..6c46dc8b19 --- /dev/null +++ b/release-notes/6.1/6.1.7.md @@ -0,0 +1,103 @@ +# Release Notes + +## Stable Release 6.1.7 - 2026-09-10 + +This update brings the following changes since the [6.1.6](6.1.6.md) release: + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4598](https://github.com/dotnet/SqlClient/pull/4598)) + +### Fixed + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4584](https://github.com/dotnet/SqlClient/pull/4584)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4552](https://github.com/dotnet/SqlClient/pull/4552)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed token authentication state handling so clearing `SqlConnection.AccessToken` preserves an existing `AccessTokenCallback` in the connection pool key, and clearing `AccessTokenCallback` preserves an existing `AccessToken`. Callback-based authentication now also follows the same prelogin server-certificate validation rules as an explicitly supplied access token. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4664](https://github.com/dotnet/SqlClient/pull/4664)) + +## Target Platform Support + +- .NET Framework 4.6.2+ (Windows x86, Windows x64, Windows ARM64) +- .NET 8.0+ (Windows x86, Windows x64, Windows ARM64, Linux, macOS) +- .NET Standard 2.0+ (Windows x86, Windows x64, Windows ARM64, Linux, macOS) + +### Dependencies + +#### .NET Framework 4.6.2 + +- Azure.Core 1.50.0 +- Azure.Identity 1.17.1 +- Microsoft.Data.SqlClient.SNI 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 +- Microsoft.IdentityModel.JsonWebTokens 7.7.1 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 7.7.1 +- System.Buffers 4.6.1 +- System.Data.Common 4.3.0 +- System.Diagnostics.DiagnosticSource 8.0.1 +- System.IdentityModel.Tokens.Jwt 7.7.1 +- System.Memory 4.6.3 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 8.0.6 +- System.Text.RegularExpressions 4.3.1 + +#### .NET 8.0 + +- Azure.Core 1.50.0 +- Azure.Identity 1.17.1 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 +- Microsoft.IdentityModel.JsonWebTokens 7.7.1 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 7.7.1 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Diagnostics.DiagnosticSource 8.0.1 +- System.IdentityModel.Tokens.Jwt 7.7.1 +- System.Security.Cryptography.Pkcs 8.0.1 + +#### .NET 9.0 + +- Azure.Core 1.50.0 +- Azure.Identity 1.17.1 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 9.0.11 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 +- Microsoft.IdentityModel.JsonWebTokens 7.7.1 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 7.7.1 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 9.0.11 +- System.Diagnostics.DiagnosticSource 9.0.11 +- System.IdentityModel.Tokens.Jwt 7.7.1 +- System.Security.Cryptography.Pkcs 9.0.11 + +#### .NET Standard 2.0 + +- Azure.Core 1.50.0 +- Azure.Identity 1.17.1 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 +- Microsoft.IdentityModel.JsonWebTokens 7.7.1 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 7.7.1 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Diagnostics.DiagnosticSource 8.0.1 +- System.IdentityModel.Tokens.Jwt 7.7.1 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 8.0.6 diff --git a/release-notes/6.1/README.md b/release-notes/6.1/README.md index e119d5baf1..b0e4a4be80 100644 --- a/release-notes/6.1/README.md +++ b/release-notes/6.1/README.md @@ -4,6 +4,7 @@ The following Microsoft.Data.SqlClient 6.1 stable releases have been shipped: | Release Date | Version | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 6.1.7 | [Release Notes](6.1.7.md) | | 2026-06-24 | 6.1.6 | [Release Notes](6.1.6.md) | | 2026-04-27 | 6.1.5 | [Release Notes](6.1.5.md) | | 2026-01-15 | 6.1.4 | [Release Notes](6.1.4.md) | diff --git a/release-notes/7.0/7.0.3.md b/release-notes/7.0/7.0.3.md new file mode 100644 index 0000000000..01dfdc6c03 --- /dev/null +++ b/release-notes/7.0/7.0.3.md @@ -0,0 +1,115 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +This update brings the following changes since the [7.0.2](7.0.2.md) release: + +The core driver and its companion packages ship together as version `7.0.3`. Update the companion packages you use alongside the driver to `7.0.3`. Assembly versions remain `7.0.0.0`, unchanged from `7.0.2`. + +### Companion package release notes + +- [Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.0.3](../add-ons/AzureKeyVaultProvider/7.0/7.0.3.md) +- [Microsoft.Data.SqlClient.Extensions.Azure 7.0.3](../Extensions/Azure/7.0/7.0.3.md) — includes the Entra ID authority parsing fix for Dataverse/Dynamics 365 connections. +- [Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3](../Extensions/Abstractions/7.0/7.0.3.md) +- [Microsoft.Data.SqlClient.Internal.Logging 7.0.3](../Internal/Logging/7.0/7.0.3.md) + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4599](https://github.com/dotnet/SqlClient/pull/4599)) + +### Fixed + +- Fixed a `SqlBulkCopy` regression in environments where the application login cannot read `sys.all_columns`. Bulk copy now falls back to the earlier column-discovery behavior when that permission is unavailable. Support for hidden columns and SQL Graph column aliases still requires access to the metadata view. + ([#4370](https://github.com/dotnet/SqlClient/issues/4370), [#4306](https://github.com/dotnet/SqlClient/pull/4306), [#4402](https://github.com/dotnet/SqlClient/pull/4402)) + +- Fixed a memory-allocation regression in connection and command operations caused by formatting diagnostic strings even when tracing was disabled. Also corrected trace messages that reported an incorrect object ID or could throw `FormatException` when traced values contained braces. + ([#4528](https://github.com/dotnet/SqlClient/pull/4528), [#4533](https://github.com/dotnet/SqlClient/pull/4533)) + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4583](https://github.com/dotnet/SqlClient/pull/4583)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4553](https://github.com/dotnet/SqlClient/pull/4553)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561)) + +- Fixed authentication state handling so clearing `SqlConnection.AccessToken`, `AccessTokenCallback`, or `SspiContextProvider` preserves the other authentication values in the connection pool key. Cloning a connection or updating its credential also preserves its `SspiContextProvider`. Combining a non-null `SspiContextProvider` with `AccessToken` or `AccessTokenCallback` now throws `InvalidOperationException` instead of silently discarding authentication state; applications must use one authentication mechanism at a time. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561), [#4644](https://github.com/dotnet/SqlClient/pull/4644)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4663](https://github.com/dotnet/SqlClient/pull/4663)) + +## Contributors + +We thank the following public contributors. Their efforts toward this project are very much appreciated. + +- [edwardneal](https://github.com/edwardneal) + +## Target Platform Support + +- .NET Framework 4.6.2+ (Windows x86, Windows x64, Windows ARM64) +- .NET 8.0+ (Windows x86, Windows x64, Windows ARM, Windows ARM64, Linux, macOS) + +### Dependencies + +#### .NET 9.0 + +- Microsoft.Bcl.Cryptography 9.0.13 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 9.0.13 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 9.0.13 +- System.Security.Cryptography.Pkcs 9.0.13 + +#### .NET 8.0 + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Security.Cryptography.Pkcs 8.0.1 + +#### .NET Standard 2.0 + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 10.0.3 +- System.Threading.Channels 10.0.3 + +#### .NET Framework 4.6.2+ + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Data.SqlClient.SNI 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- System.Buffers 4.6.1 +- System.Data.Common 4.3.0 +- System.Diagnostics.DiagnosticSource 10.0.3 +- System.Memory 4.6.3 +- System.Runtime.InteropServices.RuntimeInformation 4.3.0 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 10.0.3 +- System.Threading.Channels 10.0.3 +- System.ValueTuple 4.6.2 diff --git a/release-notes/7.0/README.md b/release-notes/7.0/README.md index aad84057e1..3057122559 100644 --- a/release-notes/7.0/README.md +++ b/release-notes/7.0/README.md @@ -4,6 +4,7 @@ The following Microsoft.Data.SqlClient 7.0 releases have been shipped: | Release Date | Version | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | | 2026-04-23 | 7.0.1 | [Release Notes](7.0.1.md) | | 2026-03-17 | 7.0.0 | [Release Notes](7.0.0.md) | diff --git a/release-notes/Extensions/Abstractions/7.0/7.0.3.md b/release-notes/Extensions/Abstractions/7.0/7.0.3.md new file mode 100644 index 0000000000..6676a750aa --- /dev/null +++ b/release-notes/Extensions/Abstractions/7.0/7.0.3.md @@ -0,0 +1,17 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +This release version-aligns `Microsoft.Data.SqlClient.Extensions.Abstractions` with the core [Microsoft.Data.SqlClient 7.0.3](../../../7.0/7.0.3.md) driver. The previous release of this package was [7.0.2](7.0.2.md). + +There are no functional or API changes in this release. Its `AssemblyVersion` remains `7.0.0.0`, unchanged from `7.0.2`. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 diff --git a/release-notes/Extensions/Abstractions/7.0/README.md b/release-notes/Extensions/Abstractions/7.0/README.md index 24f9eff90a..37fc312f2e 100644 --- a/release-notes/Extensions/Abstractions/7.0/README.md +++ b/release-notes/Extensions/Abstractions/7.0/README.md @@ -7,4 +7,5 @@ The following `Microsoft.Data.SqlClient.Extensions.Abstractions` | Release Date | Description | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | diff --git a/release-notes/Extensions/Azure/7.0/7.0.3.md b/release-notes/Extensions/Azure/7.0/7.0.3.md new file mode 100644 index 0000000000..436930ac89 --- /dev/null +++ b/release-notes/Extensions/Azure/7.0/7.0.3.md @@ -0,0 +1,39 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +This update brings the following changes since the [7.0.2](7.0.2.md) release of `Microsoft.Data.SqlClient.Extensions.Azure`. + +This package ships version-aligned with the core [Microsoft.Data.SqlClient 7.0.3](../../../7.0/7.0.3.md) driver. Its `AssemblyVersion` remains `7.0.0.0`, unchanged from `7.0.2`. + +### Fixed + +- Fixed Entra ID authentication failures when the server supplies an authority URL with an OAuth endpoint suffix, such as `https://login.microsoftonline.com//oauth2/authorize`. The provider now extracts the tenant from the first path segment and normalizes the authority used by Azure.Identity and MSAL, restoring authentication to Dataverse/Dynamics 365 TDS endpoints, including `Active Directory Service Principal` authentication. Malformed authority URLs now produce a clear authentication error. Bare tenant authority URLs used by Azure SQL and Fabric retain their existing behavior. + ([#4496](https://github.com/dotnet/SqlClient/issues/4496), [#4521](https://github.com/dotnet/SqlClient/pull/4521), [#4572](https://github.com/dotnet/SqlClient/pull/4572), [#4641](https://github.com/dotnet/SqlClient/pull/4641)) + +## Target Platform Support + +- .NET Standard 2.0 +- .NET Framework 4.6.2+ + +### Dependencies + +#### .NET Standard 2.0 + +- Azure.Core 1.51.1 +- Azure.Identity 1.18.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 + +#### .NET Framework 4.6.2+ + +- Azure.Core 1.51.1 +- Azure.Identity 1.18.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 diff --git a/release-notes/Extensions/Azure/7.0/README.md b/release-notes/Extensions/Azure/7.0/README.md index 93b90c690d..b462b02a57 100644 --- a/release-notes/Extensions/Azure/7.0/README.md +++ b/release-notes/Extensions/Azure/7.0/README.md @@ -7,4 +7,5 @@ The following `Microsoft.Data.SqlClient.Extensions.Azure` | Release Date | Description | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | diff --git a/release-notes/Internal/Logging/7.0/7.0.3.md b/release-notes/Internal/Logging/7.0/7.0.3.md new file mode 100644 index 0000000000..57dfbd1610 --- /dev/null +++ b/release-notes/Internal/Logging/7.0/7.0.3.md @@ -0,0 +1,20 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +> **Note:** This package is for internal use by other Microsoft.Data.SqlClient packages only +> and should not be referenced directly by application code. + +This release version-aligns `Microsoft.Data.SqlClient.Internal.Logging` with the core [Microsoft.Data.SqlClient 7.0.3](../../../7.0/7.0.3.md) driver. The previous release of this package was [7.0.2](7.0.2.md). + +There are no functional or API changes in this release. Its `AssemblyVersion` remains `7.0.0.0`, unchanged from `7.0.2`. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- None diff --git a/release-notes/Internal/Logging/7.0/README.md b/release-notes/Internal/Logging/7.0/README.md index 403c61db07..c4a5c3253b 100644 --- a/release-notes/Internal/Logging/7.0/README.md +++ b/release-notes/Internal/Logging/7.0/README.md @@ -10,4 +10,5 @@ The following `Microsoft.Data.SqlClient.Internal.Logging` | Release Date | Description | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | diff --git a/release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md b/release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md new file mode 100644 index 0000000000..1864b0ce42 --- /dev/null +++ b/release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md @@ -0,0 +1,21 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +This release version-aligns `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` with the core [Microsoft.Data.SqlClient 7.0.3](../../../7.0/7.0.3.md) driver. The previous release of this package was [7.0.2](7.0.2.md). + +There are no functional or API changes in this release. Its `AssemblyVersion` remains `7.0.0.0`, unchanged from `7.0.2`. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- Azure.Core 1.51.1 +- Azure.Security.KeyVault.Keys 4.9.0 +- Microsoft.Data.SqlClient 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 diff --git a/release-notes/add-ons/AzureKeyVaultProvider/7.0/README.md b/release-notes/add-ons/AzureKeyVaultProvider/7.0/README.md index a51fe94635..febedc433b 100644 --- a/release-notes/add-ons/AzureKeyVaultProvider/7.0/README.md +++ b/release-notes/add-ons/AzureKeyVaultProvider/7.0/README.md @@ -5,6 +5,7 @@ The following `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` | Release Date | Description | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | | 2026-03-17 | 7.0.0 | [Release Notes](7.0.0.md) | | 2026-03-05 | 7.0.0-preview1.26064.3 | [Release Notes](7.0.0-preview1.md) | From e16e33ed9308efc7e6a21a1a9b376ccc44141532 Mon Sep 17 00:00:00 2001 From: Mahdigln <139457032+Mahdigln@users.noreply.github.com> Date: Tue, 15 Sep 2026 02:04:59 +0330 Subject: [PATCH 39/51] Fix | GetSchema("DataTypes") does not report json type on Azure SQL (#4592) (#4682) --- .../SqlClient/SqlMetaDataFactory.DataTypes.cs | 8 +- .../Data/SqlClient/SqlMetaDataFactory.cs | 4 +- .../SqlMetaDataFactoryDataTypesTest.cs | 86 +++++++++++++++++++ 3 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataFactoryDataTypesTest.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs index eba61969c3..55b3363261 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs @@ -13,7 +13,7 @@ namespace Microsoft.Data.SqlClient; internal sealed partial class SqlMetaDataFactory { - private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet) + private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet, bool jsonTypeSupported) { DataTable dataTypesDataTable = CreateDataTypesDataTable(); @@ -58,8 +58,10 @@ private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet) minimumVersion: "10.00.000.0"); AddLongStringOrBinaryType(SqlDbType.Xml); - AddLongStringOrBinaryType(SqlDbTypeExtensions.Json, literalPrefix: "'", literalSuffix: "'", - minimumVersion: "17.00.000.0"); + if (jsonTypeSupported) + { + AddLongStringOrBinaryType(SqlDbTypeExtensions.Json, literalPrefix: "'", literalSuffix: "'"); + } AddLongStringOrBinaryType(SqlDbType.Text, literalPrefix: "'", literalSuffix: "'"); AddLongStringOrBinaryType(SqlDbType.NText, literalPrefix: "N'", literalSuffix: "'"); AddLongStringOrBinaryType(SqlDbType.Image, literalPrefix: "0x"); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs index bea2fa7e41..83eb6d857a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs @@ -39,6 +39,7 @@ internal sealed partial class SqlMetaDataFactory : IDisposable private readonly DataSet _collectionDataSet; private readonly string _serverVersion; + private readonly bool _jsonTypeSupported; public SqlMetaDataFactory(Stream xmlStream, ConnectionCapabilities connectionCapabilities) { @@ -47,6 +48,7 @@ public SqlMetaDataFactory(Stream xmlStream, ConnectionCapabilities connectionCap ADP.CheckArgumentNull(connectionCapabilities.ServerVersion, nameof(connectionCapabilities.ServerVersion)); _serverVersion = connectionCapabilities.ServerVersion; + _jsonTypeSupported = connectionCapabilities.JsonType; _collectionDataSet = LoadDataSetFromXml(xmlStream); } @@ -711,7 +713,7 @@ private DataSet LoadDataSetFromXml(Stream XmlStream) Locale = CultureInfo.InvariantCulture }; - LoadDataTypesDataTables(metaDataCollectionsDataSet); + LoadDataTypesDataTables(metaDataCollectionsDataSet, _jsonTypeSupported); XmlReaderSettings settings = new() { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataFactoryDataTypesTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataFactoryDataTypesTest.cs new file mode 100644 index 0000000000..4a0742e17d --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataFactoryDataTypesTest.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Data; +using System.Data.Common; +using System.IO; +using System.Linq; +using System.Reflection; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests; + +/// +/// Tests that correctly reports the json data type +/// in the DataTypes schema collection based on . +/// +public sealed class SqlMetaDataFactoryDataTypesTest +{ + private static Stream GetMetaDataXmlStream() + { + Assembly assembly = typeof(SqlConnection).Assembly; + Stream? stream = assembly.GetManifestResourceStream("Microsoft.Data.SqlClient.SqlMetaData.xml"); + Assert.NotNull(stream); + return stream; + } + + private static SqlMetaDataFactory CreateFactory(bool jsonTypeSupported) + { + Stream stream = GetMetaDataXmlStream(); + ConnectionCapabilities capabilities = new() + { + TdsVersion = TdsEnums.TDS7X_VERSION, + ServerMajorVersion = 12, + ServerMinorVersion = 0, + ServerBuildNumber = 0, + JsonType = jsonTypeSupported + }; + return new SqlMetaDataFactory(stream, capabilities); + } + + private static bool DataTypesContainsJson(SqlMetaDataFactory factory) + { + FieldInfo? field = typeof(SqlMetaDataFactory) + .GetField("_collectionDataSet", BindingFlags.NonPublic | BindingFlags.Instance); + Assert.NotNull(field); + + DataSet? dataSet = (DataSet?)field.GetValue(factory); + Assert.NotNull(dataSet); + + DataTable? dataTypes = dataSet.Tables[DbMetaDataCollectionNames.DataTypes]; + Assert.NotNull(dataTypes); + + DataColumn? typeNameColumn = dataTypes.Columns[DbMetaDataColumnNames.TypeName]; + Assert.NotNull(typeNameColumn); + + return dataTypes.Rows + .Cast() + .Any(row => (string)row[typeNameColumn] == "json"); + } + + /// + /// Verifies that json appears in the DataTypes schema collection when + /// is . + /// + [Fact] + public void DataTypesTable_ContainsJson_WhenJsonTypeSupported() + { + using SqlMetaDataFactory factory = CreateFactory(jsonTypeSupported: true); + + Assert.True(DataTypesContainsJson(factory)); + } + + /// + /// Verifies that json does not appear in the DataTypes schema collection when + /// is , covering + /// Azure SQL which always reports version 12.x regardless of json support. + /// + [Fact] + public void DataTypesTable_DoesNotContainJson_WhenJsonTypeNotSupported() + { + using SqlMetaDataFactory factory = CreateFactory(jsonTypeSupported: false); + + Assert.False(DataTypesContainsJson(factory)); + } +} From b67bede9545a69d53e7bd31bd35c733865290522 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:03:41 -0700 Subject: [PATCH 40/51] Bump .NET SDK to 10.0.401 (#4686) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- global.json | 4 ++-- tools/PackageCompatibility/global.json | 2 +- tools/PackageValidator/global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/global.json b/global.json index 650a5c0ddf..340837b5af 100644 --- a/global.json +++ b/global.json @@ -2,9 +2,9 @@ "sdk": { // We currently require the .NET 10 SDK to build and test the project. - "version": "10.0.300", + "version": "10.0.401", - // Allow roll-forward within the 10.0.3xx feature band so servicing patches + // Allow roll-forward within the 10.0.4xx feature band so servicing patches // are picked up automatically without requiring a PR for each bump. "rollForward": "patch", diff --git a/tools/PackageCompatibility/global.json b/tools/PackageCompatibility/global.json index af3f080d1f..78c9dbe651 100644 --- a/tools/PackageCompatibility/global.json +++ b/tools/PackageCompatibility/global.json @@ -8,7 +8,7 @@ // global.json lookup stops at the nearest file and does not merge with the repo-root file. // This subtree therefore needs to repeat the root SDK settings so commands run from // tools/PackageCompatibility keep using the same SDK behavior. - "version": "10.0.300", + "version": "10.0.401", "rollForward": "patch", "allowPrerelease": false } diff --git a/tools/PackageValidator/global.json b/tools/PackageValidator/global.json index a4b67c1ed4..bbf9d2a793 100644 --- a/tools/PackageValidator/global.json +++ b/tools/PackageValidator/global.json @@ -8,7 +8,7 @@ // global.json lookup stops at the nearest file and does not merge with the repo-root file. // This subtree therefore needs to repeat the root SDK settings so commands run from // tools/PackageValidator keep using the same SDK behavior. - "version": "10.0.300", + "version": "10.0.401", "rollForward": "patch", "allowPrerelease": false } From 8387aebd726ef59951127638bdbe2e277a546bca Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:08:43 -0700 Subject: [PATCH 41/51] Move roadmap references to the GitHub wiki (#4681) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CONTRIBUTING.md | 4 +-- README.md | 3 +- roadmap.md | 92 ------------------------------------------------- 3 files changed, 4 insertions(+), 95 deletions(-) delete mode 100644 roadmap.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 13ee1ef83b..df139c5482 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -132,7 +132,7 @@ Security issues and bugs should be reported privately, via email, to the Microso ## Submitting Pull Requests - **New features from community PRs must be driven by creating a GitHub issue first.** Discuss the proposal in the issue before starting implementation. This helps avoid wasted effort and ensures alignment with project goals. -- **Community contributions must not derail the project roadmap.** We prioritize features and fixes according to our published milestones. PRs that conflict with or distract from active roadmap items may be deferred. +- **Community contributions must align with project priorities.** We prioritize features and fixes according to our [published milestones](https://github.com/dotnet/SqlClient/milestones). PRs that conflict with or distract from active work may be deferred. - **Our maintainers reserve the right to reject PRs** that do not meet the required criteria to qualify for review. This includes PRs that: - Lack a corresponding approved issue (i.e., an issue that has been reviewed, acknowledged, and agreed upon by maintainers — typically indicated by the **`PM Approved`** field being set to **`Approved`** in the GitHub Project board) - Introduce breaking changes without prior discussion @@ -171,7 +171,7 @@ Some important caveats: ## Contribution Standards -Project maintainers will merge changes that improve the product significantly and broadly and that align with the [Microsoft.Data.SqlClient roadmap](roadmap.md). +Project maintainers will merge changes that improve the product significantly and broadly and that align with the [Microsoft.Data.SqlClient roadmap](https://github.com/dotnet/SqlClient/wiki/Roadmap). ### Requirements diff --git a/README.md b/README.md index 07374c727b..bf77802f15 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,9 @@ When targeting .NET on Windows, a package reference to [Microsoft.Data.SqlClient | Coding Style | [coding-style.md](/policy/coding-style.md) | | Contributing | [CONTRIBUTING.md](CONTRIBUTING.md) | | Copyright Information | [COPYRIGHT.md](COPYRIGHT.md) | -| Roadmap | [roadmap.md](roadmap.md) | +| Release Milestones | [GitHub milestones](https://github.com/dotnet/SqlClient/milestones) | | Review Process | [review-process.md](/policy/review-process.md) | +| Roadmap | [SqlClient roadmap](https://github.com/dotnet/SqlClient/wiki/Roadmap) | | Support Policy | [SUPPORT.md](SUPPORT.md) | ## Our Featured Contributors diff --git a/roadmap.md b/roadmap.md deleted file mode 100644 index e41d19dc79..0000000000 --- a/roadmap.md +++ /dev/null @@ -1,92 +0,0 @@ -# Microsoft.Data.SqlClient Roadmap - -The Microsoft.Data.SqlClient roadmap communicates project priorities for evolving and extending the scope of the product. We encourage the community to work with us to improve the SqlClient driver for these scenarios and extend it for others. - -> **Last updated:** May 2026 -> -> This roadmap is a living document. Priorities and timelines may shift based on community feedback, engineering constraints, and business needs. We update this page regularly to reflect the current state of development. - ---- - -## Release Milestones - -For active release milestones, their target dates, and the changes included, see [SqlClient milestones](https://github.com/dotnet/SqlClient/milestones). - ---- - -## Current Focus Areas - -Our team is actively working on the following high-level themes. Features are tracked via [GitHub issues](https://github.com/dotnet/SqlClient/issues) where applicable — see the linked milestones for associated issue details. - -- **Active** — Currently in development -- **Planned** — Committed for a future milestone with estimated delivery -- **Backlog** — On our radar for future months, not yet scheduled - -### Performance & Reliability - -| Work Item(s) | Feature | Status | ETA | -| ------------ | ------- | ------ | --- | -| [#3356](https://github.com/dotnet/SqlClient/issues/3356) | Connection pool performance improvements | Active | July 2026 | -| N/A | Performance benchmarking suite | Active | July 2026 | -| [#422](https://github.com/dotnet/SqlClient/issues/422) | Phase 1 - Unix async performance — thread starvation in parallel `ExecuteReaderAsync` | Active | September 2026 | -| TBD | Phase 2 - Async usage analysis and optimization | Planned | — | - -### New Data Type Support - -| Work Item(s) | Feature | Status | ETA | -| ------------ | ------- | ------ | --- | -| TBD | Vector subtype support — `float16` (`Half`) | Active | August 2026 | - -### Observability & Diagnostics - -| Work Item(s) | Feature | Status | ETA | -| ------------ | ------- | ------ | --- | -| [#2210](https://github.com/dotnet/SqlClient/issues/2210) [#2211](https://github.com/dotnet/SqlClient/issues/2211) | OpenTelemetry support | Planned | — | -| N/A | Logging improvements | Planned | — | -| TBD | Integrate with / expose MSAL logging | Planned | — | - -### API Improvements - -| Work Item(s) | Feature | Status | ETA | -| ------------ | ------- | ------ | --- | -| [#2353](https://github.com/dotnet/SqlClient/issues/2353) | Expose connection encryption information to clients | Planned | September 2026 | -| [#26](https://github.com/dotnet/SqlClient/issues/26) | Throw `TaskCanceledException` instead of `SqlException` for cancellations | Planned | September 2026 | -| [#113](https://github.com/dotnet/SqlClient/issues/113) | `BeginTransactionAsync` API on `SqlConnection` | Planned | — | - -### Security & Architecture - -| Feature | Status | ETA | -| ------- | ------ | --- | -| Security hardening activities | Active | Ongoing internally | - -### AI & Developer Tooling - -| Feature | Status | ETA | -| ------- | ------ | --- | -| `System.Data.SqlClient` → `Microsoft.Data.SqlClient` migration via Modernize with Copilot | Active | September 2026 | -| Modernize SqlClient repository with AI | Active | Ongoing | - -### Engineering & Infrastructure - -| Feature | Status | ETA | -| ------- | ------ | --- | -| CI/CD pipeline redesign | Active | August 2026 | -| Add SQL Server 2025 to test matrix | Planned | August 2026 | -| Add .NET 10 to test matrix | Planned | August 2026 | -| Converting existing traditional pipelines to YAML | Active | August 2026 | -| Performance benchmarking pipeline (Internal) | Planned | September 2026 | - ---- - -## Released Versions - -- [Release Notes](release-notes/README.md) — Detailed release notes summarizing all changes and features released. -- [GitHub Releases](https://github.com/dotnet/SqlClient/releases) — NuGet packages and changelog notes for each release. - ---- - -## Community Contributions & Feedback - -For information on how to contribute, see [CONTRIBUTING.md](CONTRIBUTING.md). For details on the PR tracking workflow, see [contributing-workflow.md](contributing-workflow.md). - -The best way to give feedback is to create issues in the [dotnet/SqlClient](https://github.com/dotnet/SqlClient) repo. From 92f220956b07328054701a869f452cade51830c1 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:02:58 -0700 Subject: [PATCH 42/51] Add application identity to USERAGENT payload (V2) (#4632) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../RegisteredApplication.xml | 147 +++++++++++++ .../SqlConnection.xml | 50 +++++ .../Microsoft.Data.SqlClient.csproj | 7 + .../ref/Microsoft.Data.SqlClient.cs | 37 ++++ .../Connection/SqlConnectionInternal.cs | 11 +- .../Data/SqlClient/RegisteredApplication.cs | 41 ++++ .../SqlClient/SqlClientDriverProperties.cs | 75 +++++++ .../Microsoft/Data/SqlClient/SqlConnection.cs | 29 +++ .../Data/SqlClient/SqlConnectionFactory.cs | 6 +- .../src/Microsoft/Data/SqlClient/TdsParser.cs | 13 +- .../Data/SqlClient/TdsParserHelperClasses.cs | 1 + .../src/Microsoft/Data/SqlClient/UserAgent.cs | 150 ++++++++++++-- .../SQL/SqlCommand/SqlCommandCancelTest.cs | 8 +- .../SqlConnectionConcurrentOpenTests.cs | 19 ++ .../UnitTests/RegisteredApplicationTests.cs | 161 +++++++++++++++ .../SimulatedServerTests/ConnectionTests.cs | 59 +++++- .../RegisteredApplicationPoolTests.cs | 135 ++++++++++++ .../tests/UnitTests/UserAgentTests.cs | 193 +++++++++++++++--- 18 files changed, 1081 insertions(+), 61 deletions(-) create mode 100644 doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml create mode 100644 src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/RegisteredApplication.cs create mode 100644 src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlClientDriverProperties.cs create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/RegisteredApplicationTests.cs create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/RegisteredApplicationPoolTests.cs diff --git a/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml b/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml new file mode 100644 index 0000000000..58dc683dcb --- /dev/null +++ b/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml @@ -0,0 +1,147 @@ + + + + + Specifies the known application identifiers that Microsoft.Data.SqlClient reports for user agent telemetry. + + + + Production applications that meet the bar are welcome to reserve an identifier here. + + + Identifier reservations are as follows: + + + + 0x0001-0x7FFF: Microsoft-defined large-scale applications. + + + 0x8000-0xBFFF: Reserved for small-scale use. + + + 0xC000-0xFFFF: Public and developer use. + + + + An unregistered identifier may still be reported by casting a value to this type. + + + + + + No application identity is reported. This is the default. + + + 0 + + + + + The Microsoft Entity Framework Core SQL Server provider. + + + 1 + + + + + Microsoft Semantic Kernel. + + + 2 + + + + + Microsoft SQL Server Management Studio. + + + 3 + + + + + Microsoft SQL Server Management Objects. + + + 4 + + + + + Microsoft SQL Server Data-Tier Application Framework. + + + 5 + + + + + Microsoft SQL Tools Service. + + + 6 + + + + + Microsoft ASP.NET Core distributed SQL Server cache. + + + 7 + + + + + Microsoft Entity Framework 6 SQL Server provider. + + + 8 + + + + + Microsoft Azure Functions SQL extension. + + + 9 + + + + + Microsoft Orleans ADO.NET providers. + + + 10 + + + + + Microsoft Durable Task SQL Server provider. + + + 11 + + + + + The sqlpackage command-line tool. + + + 12 + + + sqlpackage is built on the Data-Tier Application Framework, but reports its own identifier so that + command-line use can be told apart from other callers of that framework. + + + + + Microsoft Data API builder. + + + 13 + + + + diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml index 2db0e08549..7a9af8b6b4 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml @@ -2295,6 +2295,56 @@ The following sample tries to open a connection to an invalid database to simula + + + Gets or sets the middleware application identity reported to the server for this connection. + + + A value. The default is + . + + + + Set the identity before opening the connection: + + + using Microsoft.Data.SqlClient; + + using SqlConnection connection = new(connectionString) + { + RegisteredApplication = RegisteredApplication.EntityFrameworkCore + }; + connection.Open(); + + + + The connection is opening or open. The identity is reported during login, so it must be set beforehand. + + + + This API is intended for registered applications that reserve an identifier in + . An unregistered identifier may be reported by + casting a value to that type. + + + This value is telemetry. It is supplied entirely by the client, which may report any identifier in range, + so it is not an authenticated identity and must not be used for authorization or any other security + decision. + + + The identity is sent once, during login, so it must be set before the connection is opened. + + + When pooling is enabled the value is reported only while establishing a new physical connection, and it is + not part of the pool key. A connection served from the pool therefore reports the identity of whichever + connection caused that physical connection to be created, and physical connections opened in the background + to satisfy Min Pool Size report + . Applications that mix identities over one + connection string should treat this telemetry as indicative rather than exact, or disable pooling where an + exact attribution is required. + + + Gets a string that identifies the database client. diff --git a/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj b/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj index 62a4ef14e0..705182b8d5 100644 --- a/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj +++ b/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj @@ -72,6 +72,13 @@ $(NoWarn);CS0618 + + + + <_Parameter1>true + + + - 7.1.0-preview3.26226.3 + 7.1.0 [$(SniVersion), $([MSBuild]::Add($(SniVersion.Split('.')[0]), 1)).0.0) diff --git a/src/Microsoft.Data.SqlClient/Versions.props b/src/Microsoft.Data.SqlClient/Versions.props index 38df551f3b..0e075aa544 100644 --- a/src/Microsoft.Data.SqlClient/Versions.props +++ b/src/Microsoft.Data.SqlClient/Versions.props @@ -32,7 +32,7 @@ This is the *next* version to release of the SqlClient family. Update this after a version is released. Every SqlClient family package uses this version. --> - 7.1.0-preview3 + 8.0.0-preview1 From 11201fa6dedb93791e49d3a48b8c6d64856e468a Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:20:21 -0700 Subject: [PATCH 44/51] Fix release Source Link mappings and validate package source coverage (#4710) Remove the redundant src source root that remaps tracked documents outside the Source Link map. Add package-level source coverage regression checks and document local release verification. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- BUILDGUIDE.md | 31 ++ src/Directory.Build.props | 1 - tools/PackageValidator/README.md | 35 ++ .../PackageValidator/src/AssemblyInspector.cs | 21 +- tools/PackageValidator/src/Models.Report.cs | 22 ++ tools/PackageValidator/src/PortablePdb.cs | 120 +++++++ tools/PackageValidator/src/SymbolResolver.cs | 21 +- tools/PackageValidator/src/Validator.cs | 36 ++ .../test/SourceCoverageTests.cs | 310 ++++++++++++++++++ 9 files changed, 585 insertions(+), 12 deletions(-) create mode 100644 tools/PackageValidator/test/SourceCoverageTests.cs diff --git a/BUILDGUIDE.md b/BUILDGUIDE.md index 72daa88ea8..55aaa62a4e 100644 --- a/BUILDGUIDE.md +++ b/BUILDGUIDE.md @@ -282,6 +282,37 @@ Package Microsoft.Data.SqlClient.Extensions.Azure without building it beforehand dotnet build -t:PackAzure -p:PackBuild=false ``` +### Release Source Link Symbols + +To reproduce release symbol generation locally, set `BuildForRelease` in the environment before +packing. `build.proj` launches child `dotnet` processes, so passing only +`-p:BuildForRelease=true` to the orchestrator does not enable it in those processes. + +```powershell +$env:BuildForRelease = 'true' +dotnet build build.proj -t:PackSqlClient -p:Configuration=Release +Remove-Item Env:\BuildForRelease +``` + +The `.nupkg` and matching `.snupkg` are written to +`artifacts/Microsoft.Data.SqlClient/Project-Release/`. Inspect them together with +[PackageValidator](tools/PackageValidator/README.md) or NuGet Package Explorer. + +To fail validation on missing source coverage or non-portable source paths: + +```powershell +dotnet run --project tools\PackageValidator\src\PackageValidator.csproj -- ` + artifacts\Microsoft.Data.SqlClient\Project-Release ` + --fail-on missing-source-link --fail-on untracked-source --fail-on non-deterministic-source-path +``` + +Tracked source paths in the PDBs must match the Source Link document map; generated sources must +be embedded. Keep the repository root configured in `RepositoryInfo.targets` as the source root: +adding a separate `src/` root without source-control metadata can remap tracked files to `/_1/` +while Source Link only maps `/_/`. NuGet Package Explorer reports this as +**Contains untracked sources (obj)** for both Source Link and Deterministic, even when all `obj` +sources are embedded and the compiler's deterministic flag is enabled. + ## Versioning Versioning can be accomplished by using a mix of different parameters to the `build.proj` targets: diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 857bc1b65c..d9b9a7cdc8 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -141,7 +141,6 @@ Condition="'$(SqlClientVersionsImported)' != 'true'" /> - diff --git a/tools/PackageValidator/README.md b/tools/PackageValidator/README.md index f09ded9db4..3eb3996d4d 100644 --- a/tools/PackageValidator/README.md +++ b/tools/PackageValidator/README.md @@ -28,6 +28,8 @@ For every package it opens, PackageValidator reports and validates: judged only over implementation assemblies. - **Symbols** — sibling `.snupkg` matching by debug GUID with portable-PDB checksum verification, embedded-symbol detection, and orphan/mismatch reporting. +- **Source coverage** — inspect embedded and matched portable PDBs for missing Source Link mappings, + unembedded generated/temporary sources, and non-normalized document paths, without fetching sources. - **Intrinsic rules** — severity-tagged findings for missing/mismatched symbols, unsigned/delay-signed assemblies, dependency inconsistencies, and more. - **Expected versions** — optional `--expect-*` assertions for inter-package version-match @@ -80,6 +82,9 @@ or one of the following finding **categories**: - `symbol-checksum-mismatch` - `symbol-orphan` - `symbol-duplicate` +- `missing-source-link` +- `untracked-source` +- `non-deterministic-source-path` - `delay-signed` - `unsigned` - `package-unsigned` @@ -88,6 +93,34 @@ or one of the following finding **categories**: - `unexpected-file-version` - `unexpected-assembly-version` +### Source coverage and deterministic paths + +The three source categories are warnings, and follow NuGet Package Explorer's offline checks: + +- `missing-source-link`: a document is neither embedded (Embedded Source custom debug information) + nor covered by a Source Link map. Matching is case-insensitive, supports exact keys and a single + trailing `*`, and does not download the resulting URL or verify remote source checksums. +- `untracked-source`: an **unembedded** document has no Source Link mapping, or contains an `obj`, + `temp`, or `tmp` path segment (case-insensitive, either slash separator), even when Source Link maps it. +- `non-deterministic-source-path`: a document path does not begin with `/_`, including embedded + documents. This checks path normalization, not complete build reproducibility. + +For example, `/_1/src/File.cs` is normalized but is **not** covered by a `/_/*` mapping. +Generated documents under `/_/obj/` pass when embedded. Reference and satellite assemblies without +PDBs do not receive source findings. `--no-snupkg` still checks embedded PDBs. Malformed source +metadata/maps are inspection errors (exit 1), not successful coverage checks. +Like NuGet Package Explorer, document records with nil name, language, hash algorithm, or hash +handles are not inspected. + +JSON includes per-PDB `sourceCoverage` records on each inspected binary, with document counts and +the offending paths; human-readable findings include the assembly, PDB, and document path. + +Gate these checks from the repository root (keep matching `.snupkg` files beside the packages): + +```powershell +dotnet run --project .\tools\PackageValidator\src -- --fail-on missing-source-link untracked-source non-deterministic-source-path +``` + ### `--expect-*` values Each `--expect-*` value is either `VALUE` (applies to every package in the run) or `id=VALUE` @@ -219,6 +252,8 @@ The `test/` project is an xUnit v3 suite running on Microsoft.Testing.Platform. public-key-token computation, binary classification, SemVer 2.0 range evaluation (including prerelease ordering and malformed-input rejection), the rules engine, and expected-version assertions. +Source coverage tests generate portable and embedded PDB fixtures with `MetadataBuilder`, exercising +the extra-SourceRoot mapping regression, generated sources, path normalization, and symbol matching. ```bash # From tools/PackageValidator/test diff --git a/tools/PackageValidator/src/AssemblyInspector.cs b/tools/PackageValidator/src/AssemblyInspector.cs index 35a16fcc0a..9ae9ae3db7 100644 --- a/tools/PackageValidator/src/AssemblyInspector.cs +++ b/tools/PackageValidator/src/AssemblyInspector.cs @@ -85,7 +85,8 @@ public static BinaryReport Inspect(ZipArchiveEntry entry) // Read the debug directory to learn which PDB (by GUID) this assembly was built with, // whether it already carries an embedded portable PDB, and any recorded PDB checksums. - (Guid? codeViewGuid, bool hasEmbeddedPdb, List? checksums) = ReadDebugInfo(pe); + (Guid? codeViewGuid, bool hasEmbeddedPdb, List? checksums, + List? sourceCoverage) = ReadDebugInfo(pe); // File, informational, and target-framework versions are assembly-level custom // attributes, so scan for them by attribute type name. @@ -132,6 +133,7 @@ public static BinaryReport Inspect(ZipArchiveEntry entry) CodeViewGuid = codeViewGuid, Checksums = checksums, HasEmbeddedSymbols = hasEmbeddedPdb, + SourceCoverage = sourceCoverage, }; } catch (BadImageFormatException) @@ -167,12 +169,14 @@ private static SigningStatus DetermineSigningStatus(PEReader pe, bool hasPublicK /// embeds a portable PDB, and any recorded PDB checksums. /// /// The PE reader positioned over the assembly. - /// The CodeView GUID, the embedded-PDB flag, and the recorded checksums (if any). - private static (Guid? CodeViewGuid, bool HasEmbeddedPdb, List? Checksums) ReadDebugInfo(PEReader pe) + /// The CodeView GUID, embedded-PDB flag, checksums, and embedded source coverage (if any). + private static (Guid? CodeViewGuid, bool HasEmbeddedPdb, List? Checksums, + List? SourceCoverage) ReadDebugInfo(PEReader pe) { Guid? codeViewGuid = null; bool hasEmbeddedPdb = false; List? checksums = null; + List? sourceCoverage = null; foreach (DebugDirectoryEntry entry in pe.ReadDebugDirectory()) { @@ -191,6 +195,15 @@ private static (Guid? CodeViewGuid, bool HasEmbeddedPdb, List? Chec case DebugDirectoryEntryType.EmbeddedPortablePdb: hasEmbeddedPdb = true; + try + { + using MetadataReaderProvider provider = pe.ReadEmbeddedPortablePdbDebugDirectoryData(entry); + (sourceCoverage ??= []).Add(PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "embedded")); + } + catch (BadImageFormatException ex) + { + throw new InvalidDataException("Invalid embedded portable PDB.", ex); + } break; case DebugDirectoryEntryType.PdbChecksum: @@ -211,7 +224,7 @@ private static (Guid? CodeViewGuid, bool HasEmbeddedPdb, List? Chec } } - return (codeViewGuid, hasEmbeddedPdb, checksums); + return (codeViewGuid, hasEmbeddedPdb, checksums, sourceCoverage); } /// diff --git a/tools/PackageValidator/src/Models.Report.cs b/tools/PackageValidator/src/Models.Report.cs index 23979d551b..627481f0ea 100644 --- a/tools/PackageValidator/src/Models.Report.cs +++ b/tools/PackageValidator/src/Models.Report.cs @@ -221,6 +221,9 @@ internal sealed class BinaryReport /// Gets the matching PDB's path within the symbol package, if any. public string? SymbolPackageFile { get; set; } + /// Gets source coverage for embedded and matched portable PDBs, or null when none were inspected. + public List? SourceCoverage { get; set; } + /// /// Creates a report for a native or non-assembly DLL, recording its path, native version info, /// and marking it as unmanaged. @@ -236,3 +239,22 @@ internal sealed class BinaryReport NativeVersion = nativeVersion, }; } + +/// Offline source coverage and deterministic path checks for one portable PDB. +internal sealed class PdbSourceCoverage +{ + /// Gets the symbol-package entry path, or "embedded" for an embedded portable PDB. + public required string Pdb { get; init; } + + /// Gets the number of complete document records inspected, including embedded sources. + public required int DocumentCount { get; init; } + + /// Gets documents neither embedded nor covered by a Source Link mapping. + public required List MissingSourceLinkDocuments { get; init; } + + /// Gets unembedded documents without a mapping or containing an obj, temp, or tmp segment even if mapped. + public required List UntrackedDocuments { get; init; } + + /// Gets document paths that do not begin with the deterministic /_ prefix. + public required List NonNormalizedDocuments { get; init; } +} diff --git a/tools/PackageValidator/src/PortablePdb.cs b/tools/PackageValidator/src/PortablePdb.cs index 8c71c99a0b..b803b09ceb 100644 --- a/tools/PackageValidator/src/PortablePdb.cs +++ b/tools/PackageValidator/src/PortablePdb.cs @@ -3,7 +3,9 @@ // See the LICENSE file in the project root for more information. using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; using System.Security.Cryptography; +using System.Text.Json; namespace PackageValidator; @@ -19,6 +21,124 @@ internal static class PortablePdb /// The length in bytes of the PDB id (a 16-byte GUID followed by a 4-byte stamp). private const int PdbIdLength = 20; + private static readonly Guid SourceLinkKind = new("cc110556-a091-4d38-9fec-25ab9a351a6a"); + private static readonly Guid EmbeddedSourceKind = new("0e8a571b-6926-466e-b4ad-8ab04611f5fe"); + + /// Inspects document coverage without downloading sources or decompressing embedded source content. + /// The matched or embedded portable PDB metadata reader. + /// The symbol-package entry path or "embedded". + /// The source documents that fail the offline Source Link or deterministic path checks. + /// The source metadata or Source Link map is malformed. + public static PdbSourceCoverage ReadSourceCoverage(MetadataReader reader, string pdb) + { + try + { + List mappings = ReadSourceMappings(reader); + var missing = new List(); + var untracked = new List(); + var nonNormalized = new List(); + int documentCount = 0; + foreach (DocumentHandle handle in reader.Documents) + { + Document document = reader.GetDocument(handle); + if (document.Name.IsNil || document.Language.IsNil || + document.HashAlgorithm.IsNil || document.Hash.IsNil) + { + continue; + } + documentCount++; + string path = reader.GetString(document.Name); + bool embedded = reader.GetCustomDebugInformation(handle) + .Any(h => reader.GetGuid(reader.GetCustomDebugInformation(h).Kind) == EmbeddedSourceKind); + if (!embedded) + { + bool mapped = !path.Contains('*') && mappings.Any(key => key.EndsWith('*') + ? path.StartsWith(key[..^1], StringComparison.OrdinalIgnoreCase) + : string.Equals(path, key, StringComparison.OrdinalIgnoreCase)); + if (!mapped) + { + missing.Add(path); + } + if (!mapped || path.Split(['/', '\\']).Any(segment => + segment.Equals("obj", StringComparison.OrdinalIgnoreCase) || + segment.Equals("temp", StringComparison.OrdinalIgnoreCase) || + segment.Equals("tmp", StringComparison.OrdinalIgnoreCase))) + { + untracked.Add(path); + } + } + if (!path.StartsWith("/_", StringComparison.OrdinalIgnoreCase)) + { + nonNormalized.Add(path); + } + } + return new PdbSourceCoverage + { + Pdb = pdb, + DocumentCount = documentCount, + MissingSourceLinkDocuments = missing, + UntrackedDocuments = untracked, + NonNormalizedDocuments = nonNormalized, + }; + } + catch (Exception ex) when (ex is BadImageFormatException or JsonException) + { + throw new InvalidDataException($"Invalid source metadata in PDB '{pdb}'.", ex); + } + } + + /// Reads and validates module-level Source Link patterns using NPE's exact/trailing-wildcard rules. + /// The portable PDB reader. + /// Validated document path patterns; URL reachability is deliberately not checked. + private static List ReadSourceMappings(MetadataReader reader) + { + var mappings = new List(); + foreach (CustomDebugInformationHandle handle in reader.GetCustomDebugInformation( + MetadataTokens.EntityHandle(TableIndex.Module, 1))) + { + CustomDebugInformation data = reader.GetCustomDebugInformation(handle); + if (reader.GetGuid(data.Kind) != SourceLinkKind) + { + continue; + } + using JsonDocument json = JsonDocument.Parse( + reader.GetBlobBytes(data.Value), new JsonDocumentOptions { AllowTrailingCommas = true }); + if (json.RootElement.ValueKind != JsonValueKind.Object) + { + throw new InvalidDataException("Source Link must be a JSON object."); + } + foreach (JsonProperty property in json.RootElement.EnumerateObject()) + { + if (property.Name != "documents") + { + continue; + } + if (property.Value.ValueKind != JsonValueKind.Object) + { + throw new InvalidDataException("Source Link 'documents' must be an object."); + } + foreach (JsonProperty mapping in property.Value.EnumerateObject()) + { + string key = mapping.Name; + int star = key.IndexOf('*'); + if (key.Length == 0 || (star >= 0 && star != key.Length - 1) || + mapping.Value.ValueKind != JsonValueKind.String) + { + throw new InvalidDataException("Invalid Source Link document mapping."); + } + string url = mapping.Value.GetString()!; + int urlStar = url.IndexOf('*'); + if (urlStar >= 0 && (star < 0 || urlStar != url.LastIndexOf('*'))) + { + throw new InvalidDataException("Invalid Source Link URL wildcard."); + } + mappings.Add(key); + } + } + } + return mappings; + } + /// /// Reads the debug GUID from a portable PDB. /// diff --git a/tools/PackageValidator/src/SymbolResolver.cs b/tools/PackageValidator/src/SymbolResolver.cs index aa93a09d06..7145f77a45 100644 --- a/tools/PackageValidator/src/SymbolResolver.cs +++ b/tools/PackageValidator/src/SymbolResolver.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System.IO.Compression; +using System.Reflection.Metadata; namespace PackageValidator; @@ -150,7 +151,7 @@ private static void MatchSymbolPackage( asm.HasSymbolPackageSymbols = true; asm.SymbolPackageSymbolsMatch = true; asm.SymbolPackageFile = byGuid; - asm.SymbolPackageVerifiedByChecksum = VerifyMatchedChecksum(archive, byGuid, asm); + InspectMatchedPdb(archive, byGuid, asm); matchedPdbs.Add(byGuid); } else if (pdbByPathKey.TryGetValue(StripExtension(asm.Path), out string? byPath)) @@ -201,18 +202,24 @@ private static void MatchSymbolPackage( } /// - /// Re-reads a GUID-matched PDB from the symbol package on demand and verifies it against the - /// assembly's recorded PDB checksums. Loading the bytes here (rather than retaining every PDB - /// during indexing) keeps peak memory to a single PDB. + /// Re-reads a GUID-matched PDB to verify checksums and inspect source coverage. Loading bytes + /// on demand rather than retaining all PDBs during indexing keeps peak memory to one PDB. /// /// The open symbol-package archive. /// The full archive path of the matched PDB. /// The assembly whose checksums drive verification. - /// The checksum verification result, or if the entry cannot be read. - private static bool? VerifyMatchedChecksum(ZipArchive archive, string pdbFullName, BinaryReport asm) + private static void InspectMatchedPdb(ZipArchive archive, string pdbFullName, BinaryReport asm) { ZipArchiveEntry? entry = archive.GetEntry(pdbFullName); - return entry is null ? null : VerifyChecksum(asm, ReadEntry(entry)); + if (entry is null) + { + return; + } + byte[] bytes = ReadEntry(entry); + asm.SymbolPackageVerifiedByChecksum = VerifyChecksum(asm, bytes); + using var stream = new MemoryStream(bytes, writable: false); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbStream(stream); + (asm.SourceCoverage ??= []).Add(PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), pdbFullName)); } /// diff --git a/tools/PackageValidator/src/Validator.cs b/tools/PackageValidator/src/Validator.cs index 6dd738fd42..4334ce68c8 100644 --- a/tools/PackageValidator/src/Validator.cs +++ b/tools/PackageValidator/src/Validator.cs @@ -15,6 +15,9 @@ internal static class Categories public const string SymbolChecksumMismatch = "symbol-checksum-mismatch"; public const string SymbolOrphan = "symbol-orphan"; public const string SymbolDuplicate = "symbol-duplicate"; + public const string MissingSourceLink = "missing-source-link"; + public const string UntrackedSource = "untracked-source"; + public const string NonDeterministicSourcePath = "non-deterministic-source-path"; public const string DelaySigned = "delay-signed"; public const string Unsigned = "unsigned"; public const string PackageUnsigned = "package-unsigned"; @@ -30,6 +33,7 @@ internal static class Categories SymbolOrphan, SymbolDuplicate, DelaySigned, Unsigned, PackageUnsigned, DependencyInconsistency, UnexpectedPackageVersion, UnexpectedFileVersion, UnexpectedAssemblyVersion, + MissingSourceLink, UntrackedSource, NonDeterministicSourcePath, ]; } @@ -54,12 +58,44 @@ public static void Validate(PackageReport report, VersionExpectations? expectati CheckVersionConsistency(report, findings); CheckExpectedVersions(report, expectations, findings); CheckSymbols(report, findings); + CheckSourceCoverage(report, findings); CheckSigning(report, findings); CheckPackageSignature(report, findings); report.Findings = findings.Count == 0 ? null : findings; } + /// Reports source issues only for PDBs actually inspected, leaving symbolless reference/satellite assemblies alone. + private static void CheckSourceCoverage(PackageReport report, List findings) + { + foreach (BinaryReport asm in report.Binaries.Where(b => b.IsManagedAssembly && b.SourceCoverage is not null)) + { + foreach (PdbSourceCoverage coverage in asm.SourceCoverage!) + { + AddSourceFindings(coverage.MissingSourceLinkDocuments, Categories.MissingSourceLink, + "source is neither embedded nor covered by Source Link"); + AddSourceFindings(coverage.UntrackedDocuments, Categories.UntrackedSource, + "unembedded source is unmapped or contains an obj, temp, or tmp path segment"); + AddSourceFindings(coverage.NonNormalizedDocuments, Categories.NonDeterministicSourcePath, + "source path does not begin with the deterministic '/_' prefix"); + + void AddSourceFindings(List documents, string category, string message) + { + foreach (string document in documents) + { + findings.Add(new Finding + { + Severity = Severity.Warning, + Category = category, + Target = asm.Path, + Message = $"{message}: '{document}' (PDB '{coverage.Pdb}').", + }); + } + } + } + } + } + /// /// Confirms a package's version, and its assemblies' file and assembly versions, against the /// caller-supplied expected values. Pointing every package at the same expected value provides diff --git a/tools/PackageValidator/test/SourceCoverageTests.cs b/tools/PackageValidator/test/SourceCoverageTests.cs new file mode 100644 index 0000000000..0eff81804a --- /dev/null +++ b/tools/PackageValidator/test/SourceCoverageTests.cs @@ -0,0 +1,310 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Immutable; +using System.IO.Compression; +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; +using System.Reflection.PortableExecutable; +using System.Text; +using System.Text.Json; +using Xunit; + +namespace PackageValidator.Tests; + +/// Guards Source Link coverage and reproducible source paths using real portable PDB metadata. +public class SourceCoverageTests +{ + /// Checks the extra SourceRoot regression and generated-source exceptions without fetching sources. + [Theory] + [InlineData("/_1/src/File.cs", "/_/*", false, true, true, false)] + [InlineData("/_/src/File.cs", "/_/*", false, false, false, false)] + [InlineData("/_/obj/File.g.cs", null, true, false, false, false)] + [InlineData("/_/obj/File.g.cs", "/_/*", false, false, true, false)] + [InlineData("/_/temp/File.g.cs", null, false, true, true, false)] + [InlineData("/_/TMP/File.g.cs", "/_/*", false, false, true, false)] + [InlineData(@"C:\build\obj\File.g.cs", @"C:\build\*", false, false, true, true)] + [InlineData("/home/build/src/File.cs", "/home/build/*", false, false, false, true)] + [InlineData("/home/build/obj/File.cs", null, true, false, false, true)] + [InlineData("/_/object/File.cs", "/_/*", false, false, false, false)] + [InlineData("/_/src/File.cs", "/_/SRC/file.cs", false, false, false, false)] + [InlineData("/_/src/File.cs", "/_/src/Other.cs", false, true, true, false)] + [InlineData("/_/src/File.cs", "/_/SRC/*", false, false, false, false)] + [InlineData("/_/src/File.cs", "/_/sr/*", false, true, true, false)] + [InlineData("/_/src/*.cs", "/_/*", false, true, true, false)] + public void Reads_document_coverage( + string path, string? mapKey, bool embedded, bool missing, bool untracked, bool nonNormalized) + { + string? json = mapKey is null ? null : Map(mapKey); + BlobBuilder pdb = CreatePdb(path, json, embedded); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbImage(pdb.ToImmutableArray()); + + PdbSourceCoverage coverage = PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "test.pdb"); + + Assert.Equal(1, coverage.DocumentCount); + Assert.Equal(missing ? [path] : Array.Empty(), coverage.MissingSourceLinkDocuments); + Assert.Equal(untracked ? [path] : Array.Empty(), coverage.UntrackedDocuments); + Assert.Equal(nonNormalized ? [path] : Array.Empty(), coverage.NonNormalizedDocuments); + } + + /// Invalid Source Link data remains an inspection error rather than silently passing coverage. + [Theory] + [InlineData("{")] + [InlineData("[]")] + [InlineData("{\"documents\":[]}")] + [InlineData("{\"documents\":{\"\": \"https://example.invalid/file\"}}")] + [InlineData("{\"documents\":{\"/_/*/file\": \"https://example.invalid/*\"}}")] + [InlineData("{\"documents\":{\"/_/*\": \"https://example.invalid/**\"}}")] + [InlineData("{\"documents\":{\"/_/file\": \"https://example.invalid/*\"}}")] + [InlineData("{\"documents\":{\"/_/*\": 1}}")] + public void Rejects_invalid_source_maps(string json) + { + BlobBuilder pdb = CreatePdb("/_/src/File.cs", json); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbImage(pdb.ToImmutableArray()); + Assert.Throws( + () => PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "test.pdb")); + } + + /// Source map extensibility, trailing commas, and URL suffixes follow NPE matching semantics. + [Theory] + [InlineData("{\"documents\":{\"/_/*\":\"https://example.invalid/*?raw=true\"},}")] + [InlineData("{\"documents\":{\"/_/*\":\"https://example.invalid/raw\"},\"future\":true}")] + [InlineData("{\"documents\":{\"/_/src/File.cs\":\"https://example.invalid/file\"}}")] + public void Accepts_source_map_variants(string json) + { + BlobBuilder pdb = CreatePdb("/_/src/File.cs", json); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbImage(pdb.ToImmutableArray()); + Assert.Empty(PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "test.pdb").MissingSourceLinkDocuments); + } + + /// Incomplete document records are ignored just as in NPE, rather than reported as untracked sources. + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + public void Skips_incomplete_document_records(int missingHandle) + { + BlobBuilder pdb = CreatePdb("/home/build/obj/File.cs", null, missingHandle: missingHandle); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbImage(pdb.ToImmutableArray()); + PdbSourceCoverage coverage = PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "test.pdb"); + Assert.Equal(0, coverage.DocumentCount); + Assert.Empty(coverage.MissingSourceLinkDocuments); + Assert.Empty(coverage.UntrackedDocuments); + Assert.Empty(coverage.NonNormalizedDocuments); + } + + /// Embedded PDB source coverage is inspected even when sibling symbol processing is disabled. + [Fact] + public void Embedded_pdb_reports_source_findings() + { + BlobBuilder pdb = CreatePdb("/_1/src/File.cs", Map("/_/*")); + var debug = new DebugDirectoryBuilder(); + debug.AddEmbeddedPortablePdbEntry(pdb, 0x0100); + byte[] assembly = CreateAssembly(debug); + using var stream = new MemoryStream(); + using var archive = new ZipArchive(stream, ZipArchiveMode.Update); + ZipArchiveEntry entry = archive.CreateEntry("lib/net10.0/Test.dll"); + using (Stream content = entry.Open()) + { + content.Write(assembly); + } + BinaryReport binary = AssemblyInspector.Inspect(entry); + SymbolPackageInfo symbols = SymbolResolver.Resolve("unused.nupkg", [binary], false); + PackageReport report = Package(binary, symbols); + Validator.Validate(report); + + Assert.True(binary.HasSymbols); + Finding finding = Assert.Single(report.Findings!, f => f.Category == Categories.MissingSourceLink); + Assert.Equal(binary.Path, finding.Target); + Assert.Contains("/_1/src/File.cs", finding.Message); + Assert.Equal("embedded", Assert.Single(binary.SourceCoverage!).Pdb); + } + + /// Only GUID-matched symbol-package PDBs supply source findings, including all three gate categories. + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Symbol_package_only_checks_matched_pdbs(bool matches) + { + string directory = Path.Combine(Directory.GetCurrentDirectory(), "source-coverage-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + byte[] pdb = CreatePdb(@"C:\build\obj\File.g.cs", null).ToArray(); + string packagePath = Path.Combine(directory, "Test.nupkg"); + using (ZipArchive archive = ZipFile.Open(Path.ChangeExtension(packagePath, ".snupkg"), ZipArchiveMode.Create)) + using (Stream content = archive.CreateEntry("lib/net10.0/Test.pdb").Open()) + { + content.Write(pdb); + } + var binary = new BinaryReport + { + Path = "lib/net10.0/Test.dll", + Kind = BinaryKind.Implementation, + IsManagedAssembly = true, + CodeViewGuid = matches ? PortablePdb.TryReadGuid(pdb) : Guid.NewGuid(), + }; + PackageReport report = Package(binary, SymbolResolver.Resolve(packagePath, [binary], true)); + Validator.Validate(report); + + string[] sourceCategories = + [Categories.MissingSourceLink, Categories.UntrackedSource, Categories.NonDeterministicSourcePath]; + Assert.All(sourceCategories, category => + { + Assert.Contains(category, Categories.All); + Assert.Equal(matches, report.Findings!.Any(f => f.Category == category)); + }); + Assert.Equal(matches, binary.SourceCoverage is { Count: 1 }); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + /// Preserves negative and corrected SourceRoot baselines through real nupkg/snupkg archive inspection. + [Theory] + [InlineData("/_1/src/File.cs", true)] + [InlineData("/_/src/File.cs", false)] + public void Package_inspection_detects_extra_source_root(string document, bool missing) + { + string directory = Path.Combine(Directory.GetCurrentDirectory(), "source-root-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + byte[] pdb = CreatePdb(document, Map("/_/*")).ToArray(); + var debug = new DebugDirectoryBuilder(); + debug.AddCodeViewEntry("Test.pdb", new BlobContentId(PortablePdb.TryReadGuid(pdb)!.Value, 0), 0x0100); + string packagePath = Path.Combine(directory, "Test.nupkg"); + using (ZipArchive archive = ZipFile.Open(packagePath, ZipArchiveMode.Create)) + { + using (Stream content = archive.CreateEntry("lib/net10.0/Test.dll").Open()) + { + content.Write(CreateAssembly(debug)); + } + using var manifest = new StreamWriter(archive.CreateEntry("Test.nuspec").Open()); + manifest.Write("Test1.0.0"); + } + using (ZipArchive archive = ZipFile.Open(Path.ChangeExtension(packagePath, ".snupkg"), ZipArchiveMode.Create)) + using (Stream content = archive.CreateEntry("lib/net10.0/Test.pdb").Open()) + { + content.Write(pdb); + } + + PackageReport report = PackageInspector.Inspect(packagePath, processSnupkg: true); + Validator.Validate(report); + + BinaryReport binary = Assert.Single(report.Binaries); + Assert.True(binary.SymbolPackageSymbolsMatch); + Assert.True(binary.HasSymbols); + Assert.Equal(missing, report.Findings!.Any(f => f.Category == Categories.MissingSourceLink)); + Assert.Equal(missing, report.Findings!.Any(f => f.Category == Categories.UntrackedSource)); + Assert.DoesNotContain(report.Findings!, f => + f.Category == Categories.NonDeterministicSourcePath); + if (missing) + { + Assert.Contains(document, Assert.Single( + report.Findings!, f => f.Category == Categories.MissingSourceLink).Message); + } + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + /// Reference and satellite assemblies with no PDB must not create source-coverage findings. + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Legitimate_symbolless_assemblies_are_exempt(bool reference) + { + var binary = new BinaryReport + { + Path = "ref/net10.0/Test.dll", + Kind = reference ? BinaryKind.Reference : BinaryKind.Satellite, + IsManagedAssembly = true, + }; + PackageReport report = Package(binary, SymbolResolver.Resolve("unused.nupkg", [binary], false)); + Validator.Validate(report); + Assert.Null(report.Findings); + } + + /// Serializes an exact or wildcard map with an inert URL, without network access. + /// The source path pattern. + /// Source Link JSON containing one mapping. + private static string Map(string key) => JsonSerializer.Serialize(new + { + documents = new Dictionary + { + [key] = key.EndsWith('*') ? "https://example.invalid/*" : "https://example.invalid/file", + }, + }); + + /// Builds an in-memory portable PDB with a document and optional module/document custom debug records. + /// The document path. + /// Source Link JSON, or null to omit the module record. + /// Whether to embed the document's source. + /// A document handle to omit (1: name, 2: language, 3: hash algorithm, 4: hash). + /// The serialized portable PDB. + private static BlobBuilder CreatePdb(string path, string? json, bool embedded = false, int missingHandle = 0) + { + var metadata = new MetadataBuilder(); + DocumentHandle document = metadata.AddDocument( + missingHandle == 1 ? default : metadata.GetOrAddDocumentName(path), + missingHandle == 3 ? default : metadata.GetOrAddGuid(new Guid("8829d00f-11b8-4213-878b-770e8597ac16")), + missingHandle == 4 ? default : metadata.GetOrAddBlob(System.Security.Cryptography.SHA256.HashData("//"u8)), + missingHandle == 2 ? default : metadata.GetOrAddGuid(new Guid("3f5162f8-07c6-11d3-9053-00c04fa302a1"))); + if (json is not null) + { + metadata.AddCustomDebugInformation( + MetadataTokens.EntityHandle(TableIndex.Module, 1), + metadata.GetOrAddGuid(new Guid("cc110556-a091-4d38-9fec-25ab9a351a6a")), + metadata.GetOrAddBlob(Encoding.UTF8.GetBytes(json))); + } + if (embedded) + { + metadata.AddCustomDebugInformation( + document, + metadata.GetOrAddGuid(new Guid("0e8a571b-6926-466e-b4ad-8ab04611f5fe")), + metadata.GetOrAddBlob(new byte[] { 0, 0, 0, 0, 47, 47 })); + } + var rows = new int[MetadataTokens.TableCount]; + rows[(int)TableIndex.Module] = 1; + var builder = new PortablePdbBuilder(metadata, rows.ToImmutableArray(), default); + var result = new BlobBuilder(); + builder.Serialize(result); + return result; + } + + /// Builds a metadata-only assembly with the supplied debug directory for embedded-PDB integration testing. + /// The PE debug directory to include. + /// The serialized PE image. + private static byte[] CreateAssembly(DebugDirectoryBuilder debug) + { + var metadata = new MetadataBuilder(); + metadata.AddModule(0, metadata.GetOrAddString("Test.dll"), metadata.GetOrAddGuid(Guid.NewGuid()), default, default); + metadata.AddAssembly(metadata.GetOrAddString("Test"), new Version(1, 0, 0, 0), default, default, default, AssemblyHashAlgorithm.None); + var builder = new ManagedPEBuilder( + new PEHeaderBuilder(imageCharacteristics: Characteristics.Dll | Characteristics.ExecutableImage), + new MetadataRootBuilder(metadata), new BlobBuilder(), debugDirectoryBuilder: debug); + var result = new BlobBuilder(); + builder.Serialize(result); + return result.ToArray(); + } + + /// Wraps an inspected binary for validation with unrelated package-signing findings suppressed. + /// The inspected assembly. + /// Its resolved symbols. + /// A package report ready for validation. + private static PackageReport Package(BinaryReport binary, SymbolPackageInfo symbols) => new() + { + PackageFile = "Test.nupkg", + IsSigned = true, + Binaries = [binary], + SymbolPackage = symbols, + }; +} From 671d0103b2b2e52dfdfc70e19f0d30ea265e3061 Mon Sep 17 00:00:00 2001 From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:36:50 -0300 Subject: [PATCH 45/51] [v7.1.0] Release Notes (#4699) Add GA release notes for the 7.1.0 milestone on release/7.1. - New release notes for Microsoft.Data.SqlClient 7.1.0, structured as "Changes Since 7.1.0-preview3" plus "Cumulative Changes Since 7.0.3". - New version-aligned companion notes for AlwaysEncrypted.AzureKeyVaultProvider, Extensions.Azure, Extensions.Abstractions, and Internal.Logging 7.1.0. - Update each 7.1 README, the top-level release-notes README (7.1 is now the latest stable for every family package), and CHANGELOG.md. Microsoft.SqlServer.Server is skipped: it versions independently, has no PRs in this milestone, and remains at 1.0.0. --- CHANGELOG.md | 50 +- release-notes/7.1/7.1.0-preview2.md | 2 +- release-notes/7.1/7.1.0.md | 435 ++++++++++++++++++ release-notes/7.1/README.md | 1 + .../Extensions/Abstractions/7.1/7.1.0.md | 27 ++ .../Extensions/Abstractions/7.1/README.md | 1 + release-notes/Extensions/Azure/7.1/7.1.0.md | 45 ++ release-notes/Extensions/Azure/7.1/README.md | 1 + release-notes/Internal/Logging/7.1/7.1.0.md | 30 ++ release-notes/Internal/Logging/7.1/README.md | 1 + release-notes/README.md | 10 +- .../AzureKeyVaultProvider/7.1/7.1.0.md | 58 +++ .../AzureKeyVaultProvider/7.1/README.md | 1 + 13 files changed, 655 insertions(+), 7 deletions(-) create mode 100644 release-notes/7.1/7.1.0.md create mode 100644 release-notes/Extensions/Abstractions/7.1/7.1.0.md create mode 100644 release-notes/Extensions/Azure/7.1/7.1.0.md create mode 100644 release-notes/Internal/Logging/7.1/7.1.0.md create mode 100644 release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3901428151..505801db29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,56 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) > **Note:** Releases are sorted in reverse chronological order (newest first). +## [Stable Release 7.1.0] - 2026-09-17 + +General availability of Microsoft.Data.SqlClient 7.1. The sections below list the changes since [7.1.0-preview3](release-notes/7.1/7.1.0-preview3.md). See the [7.1.0 release notes](release-notes/7.1/7.1.0.md) for the cumulative list of changes since the 7.0.3 stable release. + +### Added + +- Added a `RegisteredApplication` enum and a matching `SqlConnection.RegisteredApplication` property that let a library or tool identify itself to SQL Server through version 2 of the TDS USERAGENT feature extension. The payload also carries a new driver-owned 64-bit *Driver Properties* flag field; bit 0 reports whether connection pool V2 is enabled for the process. Application identity is client-supplied telemetry and must never be used for authorization or any other security decision. The value must be set before `Open`/`OpenAsync` and is not part of the connection pool key. + ([#3201](https://github.com/dotnet/SqlClient/issues/3201), [#4632](https://github.com/dotnet/SqlClient/pull/4632)) + +### Changed + +- `SqlConnectionStringBuilder.TransparentNetworkIPResolution` is now marked `[Obsolete]`, directing callers to `MultiSubnetFailover`. There is no runtime behavior change: connection string defaults are untouched and no new AppContext switches were introduced. The only visible effect is a `CS0618` build warning for code that references the property. + ([#4494](https://github.com/dotnet/SqlClient/issues/4494), [#4576](https://github.com/dotnet/SqlClient/pull/4576)) + +- Unified the exception message raised when conflicting token-based and SSPI authentication properties are set on the same `SqlConnection`, and documented the complete set of properties that conflict with `AccessToken`. + ([#4629](https://github.com/dotnet/SqlClient/pull/4629)) + +- Documentation corrections for `SqlDataRecord`, `SqlMetaData`, and the LCID 1033 locale name. + ([#1805](https://github.com/dotnet/SqlClient/issues/1805), + [#4440](https://github.com/dotnet/SqlClient/pull/4440), + [#4646](https://github.com/dotnet/SqlClient/pull/4646)) + +- Updated `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` to v7.1.0 (was v7.1.0-preview3.26226.3). + ([#4698](https://github.com/dotnet/SqlClient/pull/4698)) + +### Fixed + +- Fixed a pooled connection being returned to the pool in a broken state after a `TransactionScope` rollback — for example, when distributed transaction promotion fails on .NET 8+ where implicit distributed transactions are disabled by default. Connection reset now preserves the transaction when the pooled connection is either a delegated transaction root or enlisted in a transaction. + ([#4001](https://github.com/dotnet/SqlClient/issues/4001), [#4557](https://github.com/dotnet/SqlClient/pull/4557)) + +- Fixed `GetSchema("DataTypes")` never reporting the SQL Server 2025 `json` type against Azure SQL. The decision now uses the `json` support flag negotiated through the TDS `FEATUREEXTACK` token instead of a server version string comparison. + ([#4592](https://github.com/dotnet/SqlClient/issues/4592), [#4682](https://github.com/dotnet/SqlClient/pull/4682)) + +- Fixed a malformed UNC pipe path being composed for IPv6 literal server names over Named Pipes in managed SNI, which could trigger an access violation inside LSASS on Windows and force a reboot. IPv6 literals are now transcribed to their `.ipv6-literal.net` form. (net8.0/net9.0 only) + ([#4523](https://github.com/dotnet/SqlClient/issues/4523), [#4558](https://github.com/dotnet/SqlClient/pull/4558)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now installed only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547)) + +- Fixed open/close throughput regressions in the opt-in connection pool V2 (`Switch.Microsoft.Data.SqlClient.UseConnectionPoolV2`). The default pool is unaffected. + ([#4543](https://github.com/dotnet/SqlClient/pull/4543)) + +### Companion packages + +- Released `Microsoft.Data.SqlClient.Extensions.Azure` 7.1.0 with an internal Entra ID authority parsing clarification and no behavior change. See [release notes](release-notes/Extensions/Azure/7.1/7.1.0.md). +- Released version-aligned `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, and `Microsoft.Data.SqlClient.Internal.Logging` 7.1.0 with no functional or API changes since preview3. See the [Azure Key Vault provider](release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md), [Abstractions](release-notes/Extensions/Abstractions/7.1/7.1.0.md), and [Logging](release-notes/Internal/Logging/7.1/7.1.0.md) release notes. + ## [Stable Release 7.0.3] - 2026-09-10 + ### Changed - Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). @@ -227,7 +275,7 @@ See the [full release notes](release-notes/7.1/7.1.0-preview2.md) for detailed d - `SqlVector` now serializes and deserializes multibyte values as little-endian explicitly. ([#3861](https://github.com/dotnet/SqlClient/pull/3861)) -- Updated the bundled .NET 10 SDK to `10.0.300`. +- Updated the .NET 10 SDK used to build the repository to `10.0.300`. ([#4287](https://github.com/dotnet/SqlClient/pull/4287)) - Re-shipped `Microsoft.Data.SqlClient.Extensions.Azure` as `7.1.0-preview2`, adding Windows Account Manager (WAM) broker support for Entra ID authentication on Windows. See [release notes](release-notes/Extensions/Azure/7.1/7.1.0-preview2.md). diff --git a/release-notes/7.1/7.1.0-preview2.md b/release-notes/7.1/7.1.0-preview2.md index 5f4a65bb78..df45fe0d7a 100644 --- a/release-notes/7.1/7.1.0-preview2.md +++ b/release-notes/7.1/7.1.0-preview2.md @@ -108,7 +108,7 @@ This update brings the following changes since the [7.1.0-preview1](7.1.0-previe - Reduced allocations by avoiding lock acquisition on `SqlErrorCollection` counters when no errors exist, and by avoiding stack-trace materialization for expected `null`-return paths. ([#4157](https://github.com/dotnet/SqlClient/pull/4157), [#4099](https://github.com/dotnet/SqlClient/pull/4099), [#4102](https://github.com/dotnet/SqlClient/pull/4102)) - Improved `EnclaveDiffieHellmanInfo.Size` accuracy. ([#4346](https://github.com/dotnet/SqlClient/pull/4346)) - `SqlVector` now serializes and deserializes little-endian multibyte values explicitly for consistent behavior across architectures. ([#3861](https://github.com/dotnet/SqlClient/pull/3861)) -- Updated the bundled .NET 10 SDK to `10.0.300`. ([#4287](https://github.com/dotnet/SqlClient/pull/4287)) +- Updated the .NET 10 SDK used to build the repository to `10.0.300`. ([#4287](https://github.com/dotnet/SqlClient/pull/4287)) ### Fixed diff --git a/release-notes/7.1/7.1.0.md b/release-notes/7.1/7.1.0.md new file mode 100644 index 0000000000..c955e6c5c9 --- /dev/null +++ b/release-notes/7.1/7.1.0.md @@ -0,0 +1,435 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +This is the general availability release of **Microsoft.Data.SqlClient 7.1**. It closes out the `7.1` preview cycle with application identity reporting for telemetry, the deprecation of `TransparentNetworkIPResolution`, and a set of connection, transaction, and Named Pipes fixes. + +> **Important — package version alignment:** Starting with the [7.0.2](../7.0/7.0.2.md) release, the `Microsoft.Data.SqlClient` driver and its companion packages share a single aligned version. The `7.1.0` GA release continues this alignment; the following packages ship together as `7.1.0`: +> +> - `Microsoft.Data.SqlClient` +> - `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` +> - `Microsoft.Data.SqlClient.Extensions.Azure` +> - `Microsoft.Data.SqlClient.Extensions.Abstractions` +> - `Microsoft.Data.SqlClient.Internal.Logging` +> +> (`Microsoft.SqlServer.Server` continues to version independently and remains at `1.0.0`.) +> +> Applications must reference the same versions of `Microsoft.Data.SqlClient` and its extensions for best compatibility. In particular, applications that reference `Microsoft.Data.SqlClient.Extensions.Azure` must upgrade it to `7.1.0` when upgrading `Microsoft.Data.SqlClient` to `7.1.0`. +> +> **Compatibility guarantee:** All aligned assemblies ship with `FileVersion 7.1.0.x` and `AssemblyVersion 7.0.0.0`. The `AssemblyVersion` is unchanged from [7.0.2](../7.0/7.0.2.md), so upgrading from `7.0.2`, `7.0.3`, or any `7.1` preview to `7.1.0` does **not** require any new .NET Framework strong-name binding redirects. Applications upgrading from `7.0.0` or `7.0.1` should note that `Extensions.Azure`, `Extensions.Abstractions`, and `Internal.Logging` raised their `AssemblyVersion` from `1.0.0.0` to `7.0.0.0` in [7.0.2](../7.0/7.0.2.md); see those release notes for the one-time .NET Framework impact. + +### Companion package release notes + +- [Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.1.0](../add-ons/AzureKeyVaultProvider/7.1/7.1.0.md) +- [Microsoft.Data.SqlClient.Extensions.Azure 7.1.0](../Extensions/Azure/7.1/7.1.0.md) +- [Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0](../Extensions/Abstractions/7.1/7.1.0.md) +- [Microsoft.Data.SqlClient.Internal.Logging 7.1.0](../Internal/Logging/7.1/7.1.0.md) + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +### Added + +#### Application Identity in the USERAGENT Payload + +*What Changed:* + +- Added a `RegisteredApplication` enum and a matching `SqlConnection.RegisteredApplication` property that let a library or tool identify itself to SQL Server through version 2 of the TDS USERAGENT feature extension. The payload also carries a new driver-owned 64-bit *Driver Properties* flag field; bit 0 reports whether connection pool V2 is enabled for the process. Both fields are emitted as unpadded uppercase hexadecimal. + ([#3201](https://github.com/dotnet/SqlClient/issues/3201), [#4632](https://github.com/dotnet/SqlClient/pull/4632)) + +*Who Benefits:* + +- Middleware and tooling built on top of the driver — Entity Framework Core, Semantic Kernel, SQL Server Management Studio, SqlPackage, Data API Builder, and similar — can be distinguished in server-side telemetry without the driver accepting arbitrary user-supplied user-agent text. This originated as a request from the Entity Framework Core team. +- Service operators gain a more accurate picture of which client stacks are connecting, which helps when diagnosing workload-specific behavior. + +*Impact:* + +- Purely additive from the application's perspective: a newly created physical connection whose `RegisteredApplication` is unset reports `Unknown` (`0`). On the wire the field itself is new — USERAGENT payload v1 carried no application identifier, while v2 always emits one. +- Set the property before calling `Open` or `OpenAsync`. Assigning it while the connection is connecting or open throws `InvalidOperationException`. + +```c# +using var connection = new SqlConnection(connectionString); +connection.RegisteredApplication = RegisteredApplication.EntityFrameworkCore; +await connection.OpenAsync(); +``` + +- The enum is `ushort`-backed and marked `[CLSCompliant(false)]`. Values are partitioned by range: `0x0001`–`0x7FFF` for Microsoft-defined large-scale applications, `0x8000`–`0xBFFF` for small-scale use, and `0xC000`–`0xFFFF` for public/developer use. Applications that are not yet registered can cast an unassigned value from the appropriate range. +- Application identity is **client-supplied telemetry and must never be used for authorization or any other security decision.** +- The value is not part of the connection pool key. A pooled physical connection reports the application that originally created it, and background `Min Pool Size` connections report `Unknown`. Cloned connections preserve the value. + +### Changed + +#### `TransparentNetworkIPResolution` Is Now Obsolete + +*What Changed:* + +- `SqlConnectionStringBuilder.TransparentNetworkIPResolution` is now marked `[Obsolete]`. The obsoletion message directs callers to `MultiSubnetFailover` and notes that Transparent Network IP Resolution (TNIR) is a .NET Framework-only feature. + ([#4494](https://github.com/dotnet/SqlClient/issues/4494), [#4576](https://github.com/dotnet/SqlClient/pull/4576)) + +*Who Benefits:* + +- Applications still relying on TNIR get a compile-time signal to move to `MultiSubnetFailover`, which addresses the same "connect quickly across multiple DNS-resolved addresses" goal, works consistently on every supported target framework, and is the documented strategy for Always On availability group listeners. + +*Impact:* + +- **No runtime behavior change.** TNIR still defaults to `true` on .NET Framework, and `MultiSubnetFailover` still defaults to `false`. The property remains .NET Framework-only and is not exposed on modern .NET, where a connection string containing the `Transparent Network IP Resolution` keyword still throws `NotSupportedException`. No new AppContext switches were introduced. +- The only visible effect is a new `CS0618` build warning for code that references the property. Suppress it, or migrate to `MultiSubnetFailover`, at your own pace. Flipping the TNIR and `MultiSubnetFailover` defaults is deferred to a future major version. + +#### Other changes + +- Updated `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` to v7.1.0 (was v7.1.0-preview3.26226.3). + ([#4698](https://github.com/dotnet/SqlClient/pull/4698)) + +- Unified the exception message raised when conflicting token-based and SSPI authentication properties are set on the same `SqlConnection`, and documented the complete set of properties that conflict with `AccessToken`. + ([#4629](https://github.com/dotnet/SqlClient/pull/4629)) + +- Documentation corrections: + - Rewrote the `SqlDataRecord` and `SqlMetaData` documentation, which incorrectly described the SQL CLR-only `SqlContext` and `SqlPipe` types, and clarified whether `SqlDataRecord` instances can be reused. + ([#1805](https://github.com/dotnet/SqlClient/issues/1805), [#4440](https://github.com/dotnet/SqlClient/pull/4440)) + - Corrected the LCID 1033 sample comment to use the official Windows locale name `English (United States)`. + ([#4646](https://github.com/dotnet/SqlClient/pull/4646)) + +### Fixed + +- Fixed a pooled connection being returned to the pool in a broken state after a `TransactionScope` rollback — for example, when distributed transaction promotion fails on .NET 8+ where implicit distributed transactions are disabled by default. A subsequent `Open()` succeeded but `BeginTransaction()` threw `InvalidOperationException` ("the connection has been broken"). Connection reset now preserves the transaction when the pooled connection is either a delegated transaction root **or** enlisted in a transaction, instead of only the latter. + ([#4001](https://github.com/dotnet/SqlClient/issues/4001), [#4557](https://github.com/dotnet/SqlClient/pull/4557)) + +- Fixed `GetSchema("DataTypes")` never reporting the SQL Server 2025 `json` type against Azure SQL. The row was filtered by a string comparison against a minimum server version of `17.00.000.0`, which Azure SQL can never satisfy because it always reports `12.00.xxxx`. The decision now uses the `json` support flag negotiated through the TDS `FEATUREEXTACK` token, which is accurate on both Azure SQL and on-premises SQL Server 2025+. + ([#4592](https://github.com/dotnet/SqlClient/issues/4592), [#4682](https://github.com/dotnet/SqlClient/pull/4682)) + +- Fixed a malformed UNC pipe path being composed for IPv6 literal server names over Named Pipes in managed SNI (for example `Server=np:::1`, `Server=np:[::1]`, or `Server=\\::1\pipe\sql\query`). A UNC path component may not contain a colon, and handing such a path to the OS could trigger an access violation inside LSASS on Windows, forcing a reboot. IPv6 literals are now transcribed to their `.ipv6-literal.net` form as defined by [MS-DTYP 2.2.57](https://learn.microsoft.com/openspecs/windows_protocols/ms-dtyp/62e862f4-2a51-452e-8eeb-dc4ff5ee33cc), and a colon-bearing host with no valid IPv6 interpretation now fails with the standard invalid-connection-string error. Colon-free host names, LocalDB, `localhost`, `.`, and IPv6 over TCP are unaffected. (net8.0/net9.0 only — this is the managed SNI counterpart to the native SNI fix) + ([#4523](https://github.com/dotnet/SqlClient/issues/4523), [#4558](https://github.com/dotnet/SqlClient/pull/4558)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now installed only while an explicitly configured custom retry provider is being resolved and constructed, and it probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-supplied handler. (net8.0/net9.0 only — the .NET Framework path does not use `AssemblyLoadContext`) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547)) + +- Fixed open/close throughput regressions in the opt-in connection pool V2 (`Switch.Microsoft.Data.SqlClient.UseConnectionPoolV2`). Acquiring an already-pooled connection no longer dispatches to the thread pool before attempting an inline, non-blocking acquisition, no longer allocates a timer-backed `CancellationTokenSource` before it is known that the caller will wait, and no longer allocates a `Task` on the synchronous completion path. The default pool is unaffected. + ([#4543](https://github.com/dotnet/SqlClient/pull/4543)) + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +This section summarizes all changes across the `7.1` preview cycle for users upgrading from the latest `7.0` stable release. Changes that were also serviced into `7.0.1`, `7.0.2`, or `7.0.3` are omitted — they are already present for `7.0.x` users. + +### Added + +#### `SqlBatch` Support on .NET Framework (net462 only) + +*What Changed:* + +- `SqlBatch`, `SqlBatchCommand`, and the related execution methods are now available on the .NET Framework target, so the batching API spans the full supported platform matrix. + ([#3926](https://github.com/dotnet/SqlClient/pull/3926)) + +*Who Benefits:* + +- Libraries and applications that multi-target .NET Framework and modern .NET can use a single data-access surface instead of maintaining separate batching strategies. + +*Impact:* + +- Purely additive; existing `SqlCommand` code is unchanged. + +#### `SqlConnection.GetSchemaAsync` + +*What Changed:* + +- Added asynchronous overloads of `SqlConnection.GetSchema` that mirror the existing synchronous shapes and honor a supplied `CancellationToken`. The .NET Framework schema code paths were unified with the .NET implementation in the process. + ([#3005](https://github.com/dotnet/SqlClient/pull/3005)) + +*Who Benefits:* + +- Applications that enumerate database metadata as part of a request pipeline no longer block a thread on synchronous I/O. + +*Impact:* + +- Additive; existing `GetSchema(...)` calls are unchanged. + +#### Asynchronous Key Store Provider APIs for Always Encrypted + +*What Changed:* + +- Added four `virtual` asynchronous counterparts to the synchronous methods on `SqlColumnEncryptionKeyStoreProvider`: `DecryptColumnEncryptionKeyAsync`, `EncryptColumnEncryptionKeyAsync`, `SignColumnMasterKeyMetadataAsync`, and `VerifyColumnMasterKeyMetadataAsync`. Each accepts an optional `CancellationToken`, and the default implementations delegate to the existing synchronous methods. + ([#3672](https://github.com/dotnet/SqlClient/issues/3672), [#3673](https://github.com/dotnet/SqlClient/pull/3673)) + +*Who Benefits:* + +- Authors of custom key store providers backed by network-bound stores such as HSMs or cloud key vaults can implement genuinely asynchronous key operations. + +*Impact:* + +- Purely additive; existing providers compile and run unmodified. +- **Introduced, but not yet consumed by the driver.** The driver's own column encryption key resolution path still invokes the synchronous provider methods on both the synchronous and asynchronous command paths. **A future release will enable their use from the driver's own asynchronous APIs.** +- The in-box `SqlColumnEncryptionAzureKeyVaultProvider` overrides all four methods. See the [AzureKeyVaultProvider 7.1.0](../add-ons/AzureKeyVaultProvider/7.1/7.1.0.md) release notes. ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +#### Configurable Idle Connection Timeout + +*What Changed:* + +- Added a `Connection Idle Timeout` connection-string keyword and matching `SqlConnectionStringBuilder.IdleTimeout` property that let the pool evict connections whose idle time exceeds the configured value. The default is `300` seconds; `0` disables idle expiration and negative values throw `ArgumentException`. Enforcement is gated on `Switch.Microsoft.Data.SqlClient.UseLegacyIdleTimeoutBehavior`, which defaults to `true` to preserve historical pooling behavior. + ([#4295](https://github.com/dotnet/SqlClient/pull/4295)) + +*Who Benefits:* + +- Applications running against Azure SQL Database and other elastic backends can bound connection idle time to align with server-side session recycling and reduce stale-connection failures. + +*Impact:* + +- Default behavior is unchanged: the keyword is parsed but not enforced until the legacy switch is set to `false`. + +#### Connection Pool V2 Feature Completeness + +*What Changed:* + +- Substantially expanded `ChannelDbConnectionPool`, the opt-in pool behind `Switch.Microsoft.Data.SqlClient.UseConnectionPoolV2`, bringing it to parity with the default `WaitHandleDbConnectionPool`: transaction support, broken-connection replacement, background warmup and replenishment to `Min Pool Size`, idle pruning derived from `Connection Idle Timeout`, optional connection-creation rate limiting, leaked-connection reclamation, and metrics/tracing parity. + ([#4395](https://github.com/dotnet/SqlClient/pull/4395), + [#4396](https://github.com/dotnet/SqlClient/pull/4396), + [#4429](https://github.com/dotnet/SqlClient/pull/4429), + [#4452](https://github.com/dotnet/SqlClient/pull/4452), + [#4463](https://github.com/dotnet/SqlClient/pull/4463), + [#4487](https://github.com/dotnet/SqlClient/pull/4487), + [#4504](https://github.com/dotnet/SqlClient/pull/4504), + [#4529](https://github.com/dotnet/SqlClient/pull/4529), + [#4543](https://github.com/dotnet/SqlClient/pull/4543)) +- `SqlConnection.ClearPool(SqlConnection)` and `SqlConnection.ClearAllPools()` now work correctly under pool V2. + ([#4194](https://github.com/dotnet/SqlClient/pull/4194)) + +*Who Benefits:* + +- Applications evaluating the V2 pool can exercise transaction-enlisted workloads, broken-connection recovery, leaked-connection reclamation, and pool warmup, and can observe the pool through the existing performance counters and EventSource traces. + +*Impact:* + +- No change to default behavior. Connection pool V2 remains **in evaluation** and is used only when the AppContext switch is enabled. + +#### Other additions + +- Added application identity reporting through `SqlConnection.RegisteredApplication` and USERAGENT payload version 2 (see *Changes Since 7.1.0-preview3* above). + ([#4632](https://github.com/dotnet/SqlClient/pull/4632)) + +- `SqlBulkCopy` column mappings now accept the SQL Graph pseudo-column aliases `$node_id`, `$edge_id`, `$from_id`, and `$to_id` as destination column names. + ([#3677](https://github.com/dotnet/SqlClient/pull/3677)) + +- `SqlBatchCommand.CommandBehavior` is now honored inside a `SqlBatch`, and `SqlBatch.ExecuteReader` respects the `CommandBehavior` passed to it. Batches that previously set the property and relied on it being ignored will now see it applied. + ([#4125](https://github.com/dotnet/SqlClient/pull/4125)) + +- Added connection-string synonyms for better compatibility with other SQL Server drivers: `ColumnEncryption`, `ConnectTimeout`, `FailoverPartner`, `PacketSize`, and `WorkstationId`. + ([#4192](https://github.com/dotnet/SqlClient/pull/4192)) + +- Added the SQL Server 2025 `json` data type to the `DataTypes` collection returned by `SqlConnection.GetSchema`. + ([#3858](https://github.com/dotnet/SqlClient/pull/3858)) + +### Changed + +#### Connection Timeout Can Now Propagate Through the Pool + +*What Changed:* + +- Replaced raw `TimeSpan` timeouts with a shared `TimeoutTimer` across `SqlConnection.Open[Async]`, pool acquisition, and physical connection creation, so the `Connect Timeout` budget can be deducted while a request waits in the pool. Enforcement is gated on `Switch.Microsoft.Data.SqlClient.UseOverallConnectTimeoutForPoolWait`, which defaults to `false`. Introduces a dependency on `Microsoft.Bcl.TimeProvider`. + ([#4270](https://github.com/dotnet/SqlClient/pull/4270)) + +*Who Benefits:* + +- Applications that observe pool contention can opt in to having the configured `Connect Timeout` respected end-to-end instead of the budget effectively restarting when a physical connection is eventually opened. + +*Impact:* + +- Default behavior is unchanged. When the switch is enabled, `Open`/`OpenAsync` under heavy pool contention may surface timeouts sooner than before; successful opens are unaffected. + +#### `TransparentNetworkIPResolution` Is Now Obsolete + +- See *Changes Since 7.1.0-preview3* above. + ([#4494](https://github.com/dotnet/SqlClient/issues/4494), [#4576](https://github.com/dotnet/SqlClient/pull/4576)) + +#### Other changes + +- Windows-only native SNI types are now removed cleanly by the IL trimmer on Linux and macOS, and `LocalAppContextSwitches.UseManagedNetworking` is substituted for a constant. The driver builds a single OS-agnostic assembly for all platforms; NuGet package structure and contents are unchanged. + ([#4207](https://github.com/dotnet/SqlClient/pull/4207), + [#4239](https://github.com/dotnet/SqlClient/issues/4239), + [#4465](https://github.com/dotnet/SqlClient/pull/4465), + [#4474](https://github.com/dotnet/SqlClient/pull/4474)) + +- Performance and allocation improvements: + - Restored reuse of `PacketData` linked-list nodes via a bounded free list on `StateSnapshot`, returning `SqlCommand/ExecuteReaderAsync` from +120.9% allocated against the 6.1.6 baseline to +0.1%. + ([#4536](https://github.com/dotnet/SqlClient/pull/4536)) + - `SqlBulkCopy` no longer builds SQL Graph column alias mapping tables when neither the source nor destination table contains graph pseudo-columns. + ([#4535](https://github.com/dotnet/SqlClient/pull/4535)) + - Reduced allocations when sending large string values to SQL Server, when reading `SqlErrorCollection` counters with no errors present, and on expected `null`-return paths that previously materialized stack traces. + ([#4072](https://github.com/dotnet/SqlClient/pull/4072), + [#4157](https://github.com/dotnet/SqlClient/pull/4157), + [#4099](https://github.com/dotnet/SqlClient/pull/4099), + [#4102](https://github.com/dotnet/SqlClient/pull/4102)) + - Use hardcoded LCID mappings when decoding strings, avoiding repeated culture lookups. + ([#4212](https://github.com/dotnet/SqlClient/pull/4212)) + +- Internal hardening and refactoring: + - `SqlConnection` internal state transitions now use `Interlocked.CompareExchange` guards. + ([#4267](https://github.com/dotnet/SqlClient/pull/4267)) + - Removed legacy connection-options inheritance from internal APIs and refactored `ForceNewConnection` handling. + ([#4235](https://github.com/dotnet/SqlClient/pull/4235), + [#4237](https://github.com/dotnet/SqlClient/pull/4237), + [#4261](https://github.com/dotnet/SqlClient/pull/4261), + [#4415](https://github.com/dotnet/SqlClient/pull/4415)) + - Added async generic helpers to reduce duplication across sync/async code paths. + ([#4334](https://github.com/dotnet/SqlClient/pull/4334)) + +- `SqlVector` now serializes and deserializes little-endian multibyte values explicitly for consistent behavior across architectures. + ([#3861](https://github.com/dotnet/SqlClient/pull/3861)) + +- Improved `EnclaveDiffieHellmanInfo.Size` accuracy. + ([#4346](https://github.com/dotnet/SqlClient/pull/4346)) + +- Documentation corrections for `SqlDataRecord`, `SqlMetaData`, server certificate configuration, and the LCID 1033 locale name. + ([#4408](https://github.com/dotnet/SqlClient/pull/4408), + [#4440](https://github.com/dotnet/SqlClient/pull/4440), + [#4646](https://github.com/dotnet/SqlClient/pull/4646)) + +- Updated Dependencies: + - Updated `Microsoft.Bcl.Cryptography`, `Microsoft.Extensions.Caching.Memory`, `System.Configuration.ConfigurationManager`, and `System.Security.Cryptography.Pkcs` to v9.0.18 for the `net9.0` target framework. Non-`net9.0` targets keep their existing `8.0.x` pins. + ([#4507](https://github.com/dotnet/SqlClient/pull/4507)) + - Added `System.Threading.RateLimiting` and `Microsoft.Bcl.TimeProvider` to the packaged dependency metadata. + ([#4270](https://github.com/dotnet/SqlClient/pull/4270), + [#4507](https://github.com/dotnet/SqlClient/pull/4507)) + - Updated `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` to v7.1.0 (was v6.0.3). + ([#4564](https://github.com/dotnet/SqlClient/pull/4564), + [#4698](https://github.com/dotnet/SqlClient/pull/4698)) + +### Fixed + +- Fixed Always Encrypted reads of `CekMdVersion` and `EkValueCount` to align with the TDS specification. + ([#4240](https://github.com/dotnet/SqlClient/pull/4240)) + +- Fixed an `OverflowException` when sending large `decimal` values (for example `decimal.MaxValue`) as a parameter with explicit `Precision` and `Scale`. This primarily affected Always Encrypted scenarios, where both must always be set. + ([#1655](https://github.com/dotnet/SqlClient/issues/1655), [#4443](https://github.com/dotnet/SqlClient/pull/4443)) + +- Fixed a TDS stream error when passing a `DateOnly` value as a parameter with `SqlDbType.Variant`, and fixed `DateOnly` values written to a `sql_variant` column of a table-valued parameter being sent as `datetime` instead of `date` (which also caused overflows for values valid for `date` but out of range for `datetime`). Reading continues to return `DateTime` instances by default for backwards compatibility. (net8.0/net9.0 only — .NET Framework has no `DateOnly` type) + ([#3953](https://github.com/dotnet/SqlClient/issues/3953), + [#3934](https://github.com/dotnet/SqlClient/issues/3934), + [#4294](https://github.com/dotnet/SqlClient/pull/4294), + [#4439](https://github.com/dotnet/SqlClient/pull/4439)) + +- Fixed a `SqlConnectionFactory` timer that woke the process every 30 seconds for the lifetime of the application even when no connection pools existed — including with `Pooling=False` and after `ClearAllPools()`. The pruning timer is now armed on demand and disarmed once there is nothing left to prune. A missing .NET Framework unload hook was also added. + ([#1881](https://github.com/dotnet/SqlClient/issues/1881), [#4479](https://github.com/dotnet/SqlClient/pull/4479)) + +- Fixed connection pool performance counter defects affecting the **default** pool as well as pool V2. `active-soft-connects` and `number-of-active-connections` could go negative after a failed connection activation, and `active-soft-connects`, `active-hard-connections`, and `number-of-pooled-connections` drifted upward permanently after a broken connection was replaced. + ([#4504](https://github.com/dotnet/SqlClient/pull/4504)) + +- Fixed a pooled connection being returned to the pool in a broken state after a `TransactionScope` rollback, which caused a later `BeginTransaction()` to throw `InvalidOperationException`. + ([#4001](https://github.com/dotnet/SqlClient/issues/4001), [#4557](https://github.com/dotnet/SqlClient/pull/4557)) + +- Fixed `GetSchema("DataTypes")` never reporting the `json` type against Azure SQL. + ([#4592](https://github.com/dotnet/SqlClient/issues/4592), [#4682](https://github.com/dotnet/SqlClient/pull/4682)) + +- Fixed a malformed UNC pipe path being composed for IPv6 literal server names over Named Pipes in managed SNI, which could trigger an LSASS access violation and a forced reboot on Windows. (net8.0/net9.0 only) + ([#4523](https://github.com/dotnet/SqlClient/issues/4523), [#4558](https://github.com/dotnet/SqlClient/pull/4558)) + +- Fixed several async entry points in `SqlBulkCopy`, `SqlDataReader.InvokeAsyncCall`, `SqlCommand.Reader`, and `SqlCommand.Xml` that captured fatal exceptions such as `OutOfMemoryException` into faulted `Task`s instead of letting them propagate. + ([#4437](https://github.com/dotnet/SqlClient/pull/4437)) + +- Fixed a `SqlDataReader` streaming bug where calling `IsDBNull()` before reading a streamed value could skip column data. + ([#4082](https://github.com/dotnet/SqlClient/pull/4082)) + +- Fixed a race in `SqlConnection.TryOpenInner` that could surface as `InvalidCastException`; the same race now returns a deterministic `InvalidOperationException`. + ([#4179](https://github.com/dotnet/SqlClient/pull/4179)) + +- Fixed `LoginWithFailover` to validate parser state before continuing, preventing null-reference failures during failover login. + ([#4140](https://github.com/dotnet/SqlClient/pull/4140)) + +- Fixed the SPN used during login to use the resolved port instead of the instance name when `Protocol=None` or `Protocol=Admin` is specified. + ([#4180](https://github.com/dotnet/SqlClient/pull/4180)) + +- Fixed several `CancellationTokenSource` leaks in `SqlDataReader`, `SqlConnection`, the `SqlCommand` reconnect paths, and the sequential-stream helpers. + ([#4009](https://github.com/dotnet/SqlClient/pull/4009)) + +### Removed + +#### SQL Server 7.0 and 2000 Support + +*What Changed:* + +- Removed dead protocol-level code paths for SQL Server 7.0 and SQL Server 2000, along with the now-orphaned `SQL Server 2000` type-system compatibility option. The `TypeSystem.SQLServer2000` enum value and the `Type System Version=SQL Server 2000` connection-string branch are gone. + ([#4015](https://github.com/dotnet/SqlClient/pull/4015)) + +*Who Benefits:* + +- The driver no longer carries dead legacy code, and the documented connection-string surface (`Latest`, `SQL Server 2005`, `SQL Server 2008`, `SQL Server 2012`) now matches the implementation exactly. + +*Impact:* + +- **Breaking:** A connection string that specifies `Type System Version=SQL Server 2000` now throws `ArgumentException` when the connection is opened. Switch to a supported value such as `Latest`. There is no change to which servers the driver connects to — SQL Server 7.0 and 2000 were already rejected during login version negotiation. + +## Contributors + +We thank the following public contributors. Their efforts toward this project are very much appreciated. + +- [edwardneal](https://github.com/edwardneal) +- [Mahdigln](https://github.com/Mahdigln) + +## Target Platform Support + +- .NET Framework 4.6.2+ (Windows x86, Windows x64, Windows ARM64) +- .NET 8.0+ (Windows x86, Windows x64, Windows ARM, Windows ARM64, Linux, macOS) + +### Dependencies + +#### .NET 9.0 + +- Microsoft.Bcl.Cryptography 9.0.18 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Data.SqlClient.SNI.runtime 7.1.0 +- Microsoft.Extensions.Caching.Memory 9.0.18 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 9.0.18 +- System.Security.Cryptography.Pkcs 9.0.18 +- System.Threading.RateLimiting 9.0.18 + +#### .NET 8.0 + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Data.SqlClient.SNI.runtime 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Threading.RateLimiting 8.0.0 + +#### .NET Standard 2.0 + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Bcl.TimeProvider 8.0.1 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Data.SqlClient.SNI.runtime 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 10.0.3 +- System.Threading.Channels 10.0.3 +- System.Threading.RateLimiting 8.0.0 + +#### .NET Framework 4.6.2+ + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Bcl.TimeProvider 8.0.1 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Data.SqlClient.SNI 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- System.Buffers 4.6.1 +- System.Data.Common 4.3.0 +- System.Diagnostics.DiagnosticSource 10.0.3 +- System.Memory 4.6.3 +- System.Runtime.InteropServices.RuntimeInformation 4.3.0 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 10.0.3 +- System.Threading.Channels 10.0.3 +- System.Threading.RateLimiting 8.0.0 +- System.ValueTuple 4.6.2 diff --git a/release-notes/7.1/README.md b/release-notes/7.1/README.md index 0b7de8156d..b2c03f09f3 100644 --- a/release-notes/7.1/README.md +++ b/release-notes/7.1/README.md @@ -7,3 +7,4 @@ The following Microsoft.Data.SqlClient 7.1 releases have been shipped: | 2026-04-29 | 7.1.0-preview1 | [Release Notes](7.1.0-preview1.md) | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/release-notes/Extensions/Abstractions/7.1/7.1.0.md b/release-notes/Extensions/Abstractions/7.1/7.1.0.md new file mode 100644 index 0000000000..f5572308c6 --- /dev/null +++ b/release-notes/Extensions/Abstractions/7.1/7.1.0.md @@ -0,0 +1,27 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +This release continues version-alignment of `Microsoft.Data.SqlClient.Extensions.Abstractions` with the core `Microsoft.Data.SqlClient` driver version (`7.1.0`). The previous release of this package was [7.1.0-preview3](7.1.0-preview3.md), and the previous stable release was [7.0.3](../7.0/7.0.3.md). + +There are no functional or API changes in this release. See the core [Microsoft.Data.SqlClient 7.1.0](../../../7.1/7.1.0.md) release notes for the driver-family changes shipped alongside this version. + +> **Version alignment:** This package's version continues to track the core `Microsoft.Data.SqlClient` driver version. See the [7.0.2 release notes](../../../7.0/7.0.2.md) for the initial alignment announcement. This assembly ships with `FileVersion 7.1.0.x`. The `AssemblyVersion 7.0.0.0` remains unchanged from the `7.0.2` release, so upgrading from `7.0.2`, `7.0.3`, or any `7.1` preview does not require new .NET Framework strong-name binding redirects. This package's `AssemblyVersion` was raised from `1.0.0.0` to `7.0.0.0` in `7.0.2`, so applications moving from the earlier `1.0.0` package may still need the one-time .NET Framework binding redirect described in those notes. + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +There are no functional or API changes in this release. + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +There are no functional or API changes in this package across the `7.1` preview cycle. Every `7.1` release of this package has been a version-alignment release. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 diff --git a/release-notes/Extensions/Abstractions/7.1/README.md b/release-notes/Extensions/Abstractions/7.1/README.md index 9d1374bdb6..20ba3ecff0 100644 --- a/release-notes/Extensions/Abstractions/7.1/README.md +++ b/release-notes/Extensions/Abstractions/7.1/README.md @@ -7,3 +7,4 @@ The following `Microsoft.Data.SqlClient.Extensions.Abstractions` | :-- | :-- | :--: | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/release-notes/Extensions/Azure/7.1/7.1.0.md b/release-notes/Extensions/Azure/7.1/7.1.0.md new file mode 100644 index 0000000000..3c73bb6631 --- /dev/null +++ b/release-notes/Extensions/Azure/7.1/7.1.0.md @@ -0,0 +1,45 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +This release continues version-alignment of `Microsoft.Data.SqlClient.Extensions.Azure` with the core `Microsoft.Data.SqlClient` driver version (`7.1.0`). The previous release of this package was [7.1.0-preview3](7.1.0-preview3.md), and the previous stable release was [7.0.3](../7.0/7.0.3.md). See the core [Microsoft.Data.SqlClient 7.1.0](../../../7.1/7.1.0.md) release notes for the driver-family changes shipped alongside this version. + +> **Version alignment:** This package's version continues to track the core `Microsoft.Data.SqlClient` driver version. See the [7.0.2 release notes](../../../7.0/7.0.2.md) for the initial alignment announcement. This assembly ships with `FileVersion 7.1.0.x`. The `AssemblyVersion 7.0.0.0` remains unchanged from the `7.0.2` release, so upgrading from `7.0.2`, `7.0.3`, or any `7.1` preview does not require new .NET Framework strong-name binding redirects. This package's `AssemblyVersion` was raised from `1.0.0.0` to `7.0.0.0` in `7.0.2`, so applications moving from the earlier `1.0.0` package may still need the one-time .NET Framework binding redirect described in those notes. + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +### Changed + +- Clarified the internal Entra ID authority handling in `ActiveDirectoryAuthenticationProvider` by keeping the server-supplied STSURL, the Azure Identity authority URL, and the normalized MSAL authority distinct, and by composing the MSAL authority only on the MSAL-based code paths. There is no behavior change and no public API change. + ([#4630](https://github.com/dotnet/SqlClient/pull/4630)) + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +There are no additional cumulative changes since [7.0.3](../7.0/7.0.3.md). + +## Target Platform Support + +- .NET Standard 2.0 +- .NET Framework 4.6.2+ + +### Dependencies + +#### .NET Standard 2.0 + +- Azure.Core 1.51.1 +- Azure.Identity 1.18.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 + +#### .NET Framework 4.6.2+ + +- Azure.Core 1.51.1 +- Azure.Identity 1.18.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 diff --git a/release-notes/Extensions/Azure/7.1/README.md b/release-notes/Extensions/Azure/7.1/README.md index b3f93b7931..7cb63309db 100644 --- a/release-notes/Extensions/Azure/7.1/README.md +++ b/release-notes/Extensions/Azure/7.1/README.md @@ -7,3 +7,4 @@ The following `Microsoft.Data.SqlClient.Extensions.Azure` | :-- | :-- | :--: | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/release-notes/Internal/Logging/7.1/7.1.0.md b/release-notes/Internal/Logging/7.1/7.1.0.md new file mode 100644 index 0000000000..797e5b9050 --- /dev/null +++ b/release-notes/Internal/Logging/7.1/7.1.0.md @@ -0,0 +1,30 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +> **Note:** This package is for internal use by other Microsoft.Data.SqlClient packages only +> and should not be referenced directly by application code. + +This release continues version-alignment of `Microsoft.Data.SqlClient.Internal.Logging` with the core `Microsoft.Data.SqlClient` driver version (`7.1.0`). The previous release of this package was [7.1.0-preview3](7.1.0-preview3.md), and the previous stable release was [7.0.3](../7.0/7.0.3.md). + +There are no functional or API changes in this release. See the core [Microsoft.Data.SqlClient 7.1.0](../../../7.1/7.1.0.md) release notes for the driver-family changes shipped alongside this version. + +> **Version alignment:** This package's version continues to track the core `Microsoft.Data.SqlClient` driver version. See the [7.0.2 release notes](../../../7.0/7.0.2.md) for the initial alignment announcement. This assembly ships with `FileVersion 7.1.0.x`. The `AssemblyVersion 7.0.0.0` remains unchanged from the `7.0.2` release, so upgrading from `7.0.2`, `7.0.3`, or any `7.1` preview does not require new .NET Framework strong-name binding redirects. This package's `AssemblyVersion` was raised from `1.0.0.0` to `7.0.0.0` in `7.0.2`, so applications moving from the earlier `1.0.0` package may still need the one-time .NET Framework binding redirect described in those notes. + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +There are no functional or API changes in this release. + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +There are no functional or API changes in this package across the `7.1` preview cycle. Every `7.1` release of this package has been a version-alignment release. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- None diff --git a/release-notes/Internal/Logging/7.1/README.md b/release-notes/Internal/Logging/7.1/README.md index 25caeccaba..005fe0e118 100644 --- a/release-notes/Internal/Logging/7.1/README.md +++ b/release-notes/Internal/Logging/7.1/README.md @@ -10,3 +10,4 @@ The following `Microsoft.Data.SqlClient.Internal.Logging` | :-- | :-- | :--: | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/release-notes/README.md b/release-notes/README.md index c32e0bc2be..600cad636f 100644 --- a/release-notes/README.md +++ b/release-notes/README.md @@ -1,6 +1,6 @@ # Microsoft.Data.SqlClient Release Notes -The latest stable release is [Microsoft.Data.SqlClient 7.0](7.0). +The latest stable release is [Microsoft.Data.SqlClient 7.1](7.1). ## Release Information @@ -23,7 +23,7 @@ The latest stable release is [Microsoft.Data.SqlClient 7.0](7.0). # Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider Release Notes The latest stable release is -[Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.0](add-ons/AzureKeyVaultProvider/7.0). +[Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.1](add-ons/AzureKeyVaultProvider/7.1). ## Release Information @@ -41,7 +41,7 @@ The latest stable release is # Microsoft.Data.SqlClient.Extensions.Abstractions Release Notes The latest release is -[Microsoft.Data.SqlClient.Extensions.Abstractions 7.0](Extensions/Abstractions/7.0). +[Microsoft.Data.SqlClient.Extensions.Abstractions 7.1](Extensions/Abstractions/7.1). ## Release Information @@ -52,7 +52,7 @@ The latest release is # Microsoft.Data.SqlClient.Extensions.Azure Release Notes The latest release is -[Microsoft.Data.SqlClient.Extensions.Azure 7.0](Extensions/Azure/7.0). +[Microsoft.Data.SqlClient.Extensions.Azure 7.1](Extensions/Azure/7.1). ## Release Information @@ -64,7 +64,7 @@ The latest release is # Microsoft.Data.SqlClient.Internal.Logging Release Notes The latest release is -[Microsoft.Data.SqlClient.Internal.Logging 7.0](Internal/Logging/7.0). +[Microsoft.Data.SqlClient.Internal.Logging 7.1](Internal/Logging/7.1). ## Release Information diff --git a/release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md b/release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md new file mode 100644 index 0000000000..67d99ba296 --- /dev/null +++ b/release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md @@ -0,0 +1,58 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +This release continues version-alignment of `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` with the core `Microsoft.Data.SqlClient` driver version (`7.1.0`). The previous release of this package was [7.1.0-preview3](7.1.0-preview3.md), and the previous stable release was [7.0.3](../7.0/7.0.3.md). + +There are no functional or API changes since [7.1.0-preview3](7.1.0-preview3.md). See the core [Microsoft.Data.SqlClient 7.1.0](../../../7.1/7.1.0.md) release notes for the driver-family changes shipped alongside this version. + +> **Version alignment:** This package's version continues to track the core `Microsoft.Data.SqlClient` driver version. See the [7.0.2 release notes](../../../7.0/7.0.2.md) for the initial alignment announcement. This assembly ships with `FileVersion 7.1.0.x`. The `AssemblyVersion 7.0.0.0` remains unchanged from the `7.0.2` release, so upgrading from any `7.0.x` release does not require new .NET Framework strong-name binding redirects. + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +There are no functional or API changes in this release. + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +### Added + +#### Asynchronous key store provider APIs + +*What Changed:* + +- `SqlColumnEncryptionAzureKeyVaultProvider` now overrides the four asynchronous key store provider methods introduced on the base class in [#3673](https://github.com/dotnet/SqlClient/pull/3673): `EncryptColumnEncryptionKeyAsync`, `DecryptColumnEncryptionKeyAsync`, `SignColumnMasterKeyMetadataAsync`, and `VerifyColumnMasterKeyMetadataAsync`. These call the Azure SDK's own asynchronous methods and flow the supplied `CancellationToken` to them, rather than completing synchronous work on a returned task. +- Concurrent cache misses for the same key are gated so a burst of callers issues a single Key Vault request. The gate is only ever awaited, so no thread blocks, and misses for different keys still proceed in parallel. The gate applies only while caching is enabled — setting `ColumnEncryptionKeyCacheTtl` to zero disables caching and bypasses the gate, so concurrent operations for the same key may issue parallel Key Vault requests. + ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +*Who Benefits:* + +- Applications using Always Encrypted with Azure Key Vault can await column encryption key operations rather than blocking a thread pool thread on network-bound key vault I/O, when they call the provider directly. + +*Impact:* + +- No public API was removed or changed. The column encryption key and signature caches are shared between the synchronous and asynchronous paths, so a key resolved by one is visible to the other. +- **Introduced, but not yet consumed by the driver.** `Microsoft.Data.SqlClient`'s own key resolution path still invokes the synchronous provider methods on both synchronous and asynchronous command execution, so these overrides do not by themselves make Always Encrypted query execution non-blocking. **A future release will enable their use from the driver's own asynchronous APIs.** See the core [7.1.0](../../../7.1/7.1.0.md) release notes. +- **Behavior change:** `VerifyColumnMasterKeyMetadata` and `VerifyColumnMasterKeyMetadataAsync` now reject a null or empty `signature` with `ArgumentNullException` / `ArgumentException` instead of passing it through to the Azure SDK. In-product callers are unaffected because `SqlSecurityUtility.VerifyColumnMasterKeySignature` already rejects those values upstream. +- **Runtime requirement:** this version requires `Microsoft.Data.SqlClient` `7.1.0-preview3` or later at runtime, because the asynchronous base-class methods it overrides first shipped in that release. Pairing it with the aligned stable `7.1.0` is recommended, and the NuGet dependency floor resolves there. Because assembly versions unify at `major.0.0.0`, the provider also binds against a `7.1` preview driver, but running it against a driver older than `7.1.0-preview3` produces a `TypeLoadException`. + +### Fixed + +- Fixed the 2000-entry column master key signature cache being able to stop compacting and grow unbounded. `LocalCache.GetOrCreate` compacted only when `Count == maxSize`, and under concurrency that equality test could be stepped past, permanently disabling compaction. It now compacts on `Count >= maxSize`. + ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +- Fixed concurrent callers ending up with different `CryptographyClient` instances for the same key, which produced redundant clients and duplicated Key Vault handshakes. `GetCryptographyClient` now uses `GetOrAdd` instead of `TryGetValue` followed by `TryAdd`. + ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- Azure.Core 1.51.1 +- Azure.Security.KeyVault.Keys 4.9.0 +- Microsoft.Data.SqlClient 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 diff --git a/release-notes/add-ons/AzureKeyVaultProvider/7.1/README.md b/release-notes/add-ons/AzureKeyVaultProvider/7.1/README.md index 07fffc069b..6232931b72 100644 --- a/release-notes/add-ons/AzureKeyVaultProvider/7.1/README.md +++ b/release-notes/add-ons/AzureKeyVaultProvider/7.1/README.md @@ -7,3 +7,4 @@ The following `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` | :-- | :-- | :--: | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | From a41354935a388f0e4cb9b9fb0a8c1e37f13a7a0a Mon Sep 17 00:00:00 2001 From: Saurabh Singh Date: Tue, 22 Sep 2026 12:58:29 -0700 Subject: [PATCH 46/51] Fix precision of rescaled zero decimal parameters (#4721) --- .../src/Microsoft/Data/SqlClient/TdsParser.cs | 5 +- .../SQL/ParameterTest/ParametersTest.cs | 113 ++++++++++++- .../tests/UnitTests/TdsParserDecimalTests.cs | 156 ++++++++++++++++++ 3 files changed, 268 insertions(+), 6 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/TdsParserDecimalTests.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs index 7c079cfee0..92e40cc292 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -10433,7 +10433,10 @@ private Task TDSExecuteRPCAddParameter(TdsParserStateObject stateObj, SqlParamet // If Precision is specified, verify value precision vs param precision if (precision != 0) { - if (precision < adjustedValue.Precision) + // Precision metadata can overstate zero's required digits. + // Compare magnitudes to recognize negative zero as well. + if (precision < adjustedValue.Precision && + (SqlDecimal.Abs(adjustedValue) != new SqlDecimal(0)).IsTrue) { throw ADP.ParameterValueOutOfRange(adjustedValue); } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/ParametersTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/ParametersTest.cs index 1bb9b23824..77b58ce09b 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/ParametersTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/ParametersTest.cs @@ -532,11 +532,112 @@ public static void TestScaledDecimalParameter_CommandInsert(string connectionStr Assert.True(ValidateInsertedValues(connection, decimalTable.Name, truncateScaledDecimal), $"Invalid test happened with connection string [{connection.ConnectionString}]"); } - [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] - public static void TestOutOfRangeDecimalParameter_CommandSelect() + /// + /// CLR and SQL zero values with different representations must round-trip into decimal(p,p) columns. + /// One async case covers command execution parity; scale boundaries use the shared synchronous conversion. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTCPConnStringSetup))] + [InlineData(1, false)] + [InlineData(2, false)] + [InlineData(3, false)] + [InlineData(3, true)] + [InlineData(28, false)] + [InlineData(29, false)] + [InlineData(38, false)] + public static async Task ZeroDecimalParameter_CommandInsert(byte scale, bool useAsync) { using SqlConnection connection = new(DataTestUtility.TCPConnectionString); - connection.Open(); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + + using Table table = new(connection, "ZeroDecimalParameter", $"([Value] decimal({scale},{scale}))"); + using SqlCommand command = new( + $"INSERT INTO {table.Name} ([Value]) OUTPUT INSERTED.[Value] VALUES (@Value)", connection); + SqlParameter parameter = command.Parameters.Add("@Value", SqlDbType.Decimal); + parameter.Precision = scale; + parameter.Scale = scale; + + foreach (object value in new object[] + { + 0m, 0.0m, 0.000m, new decimal(0, 0, 0, true, 0), + new SqlDecimal(0m), new SqlDecimal(0.000m), + new SqlDecimal(new decimal(0, 0, 0, true, 0)), + new SqlDecimal(38, scale, true, 0, 0, 0, 0), + new SqlDecimal(38, scale, false, 0, 0, 0, 0) + }) + { + parameter.Value = value; + using SqlDataReader reader = useAsync + ? await command.ExecuteReaderAsync() + : command.ExecuteReader(); + Assert.True(useAsync ? await reader.ReadAsync() : reader.Read()); + SqlDecimal actual = reader.GetSqlDecimal(0); + Assert.Equal(scale, actual.Precision); + Assert.Equal(scale, actual.Scale); + Assert.Equal(0, actual.CompareTo(new SqlDecimal(0))); + } + } + + /// + /// Correcting zero precision must not allow nonzero values that exceed the parameter precision. + /// Both command APIs must surface the same exception, including when the async result is awaited. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTCPConnStringSetup))] + [InlineData(false)] + [InlineData(true)] + public static async Task DecimalParameter_RejectsInsufficientPrecision(bool useAsync) + { + using SqlConnection connection = new(DataTestUtility.TCPConnectionString); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + + using SqlCommand command = new("SELECT @Value", connection); + SqlParameter parameter = command.Parameters.Add("@Value", SqlDbType.Decimal); + parameter.Precision = 3; + parameter.Scale = 3; + foreach (object value in new object[] { 1m, -1m, new SqlDecimal(1m), new SqlDecimal(-1m) }) + { + parameter.Value = value; + if (useAsync) + { + await Assert.ThrowsAsync(() => command.ExecuteNonQueryAsync()); + } + else + { + Assert.Throws(() => command.ExecuteNonQuery()); + } + } + } + + /// + /// Large decimal values must still round-trip after rescaling beyond the CLR decimal capacity. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTCPConnStringSetup))] + [InlineData(false)] + [InlineData(true)] + public static async Task TestOutOfRangeDecimalParameter_CommandSelect(bool useAsync) + { + using SqlConnection connection = new(DataTestUtility.TCPConnectionString); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } using SqlCommand cmd = new("SELECT @Value", connection); // A System.Decimal value has a maximum precision of 29 digits. We specify a Precision of 38 and a Scale of 2 in order @@ -550,9 +651,11 @@ public static void TestOutOfRangeDecimalParameter_CommandSelect() cmd.Parameters.Add(p); - using SqlDataReader reader = cmd.ExecuteReader(); + using SqlDataReader reader = useAsync + ? await cmd.ExecuteReaderAsync() + : cmd.ExecuteReader(); - reader.Read(); + Assert.True(useAsync ? await reader.ReadAsync() : reader.Read()); // Read the original value back as a SqlDecimal, with matching scale and precision. SqlDecimal roundtrippedDecimal = reader.GetSqlDecimal(0); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/TdsParserDecimalTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/TdsParserDecimalTests.cs new file mode 100644 index 0000000000..57a4f2cfd8 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/TdsParserDecimalTests.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.Data.SqlTypes; +using System.Globalization; +using System.Reflection; +using Microsoft.Data.SqlClient.Tests.Common; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests +{ + /// + /// Verifies decimal scale adjustment and RPC precision validation preserve representable values. + /// + [Collection(AppContextSwitchTestCollection.Name)] + public class TdsParserDecimalTests + { + /// + /// CLR and SQL zero must serialize when precision equals scale, regardless of input scale or sign. + /// + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void RpcDecimalParameter_ZeroFitsPrecisionEqualToScale(bool isSqlDecimal, bool truncate) + { + using LocalAppContextSwitchesHelper switches = new(); + switches.TruncateScaledDecimal = truncate; + + for (byte oldScale = 0; oldScale <= 28; oldScale++) + { + foreach (bool negative in new[] { false, true }) + { + decimal zero = new(0, 0, 0, negative, oldScale); + object value = isSqlDecimal ? (object)new SqlDecimal(zero) : zero; + for (byte scale = 1; scale <= 38; scale++) + { + WriteDecimalParameter(value, scale, scale); + } + } + } + } + + /// + /// Excess SqlDecimal precision must not reject signed zero or a value that rounds/truncates to zero. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RpcDecimalParameter_ZeroWithExcessPrecisionFits(bool negative) + { + using LocalAppContextSwitchesHelper switches = new(); + foreach (bool truncate in new[] { false, true }) + { + switches.TruncateScaledDecimal = truncate; + WriteDecimalParameter(new SqlDecimal(38, 3, !negative, 0, 0, 0, 0), 3, 3); + WriteDecimalParameter(SqlDecimal.ConvertToPrecScale( + new SqlDecimal(new decimal(0, 0, 0, negative, 3)), 38, 3), 3, 3); + WriteDecimalParameter(new SqlDecimal(38, 4, !negative, 1, 0, 0, 0), 3, 3); + } + } + + /// + /// Exempting zero must not permit nonzero CLR or SQL values with insufficient parameter precision. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RpcDecimalParameter_NonzeroExceedingPrecisionThrows(bool isSqlDecimal) + { + foreach (decimal value in new[] { -1m, 1m }) + { + object parameterValue = isSqlDecimal ? (object)new SqlDecimal(value) : value; + TargetInvocationException exception = Assert.Throws( + () => WriteDecimalParameter(parameterValue, 3, 3)); + Assert.IsType(exception.InnerException); + } + } + + /// + /// Scale adjustment must retain the rounding switch and precision needed for nonzero values. + /// + [Theory] + [InlineData("0.001", 3, "0.001", "0.001", 3)] + [InlineData("0.01", 3, "0.010", "0.010", 3)] + [InlineData("1", 3, "1.000", "1.000", 4)] + [InlineData("-1", 3, "-1.000", "-1.000", 4)] + [InlineData("1.005", 2, "1.01", "1.00", 3)] + [InlineData("-1.005", 2, "-1.01", "-1.00", 3)] + public void AdjustDecimalScale_NonzeroPreservesValueAndPrecision( + string value, int scale, string rounded, string truncated, int precision) + { + using LocalAppContextSwitchesHelper switches = new(); + foreach (bool truncate in new[] { false, true }) + { + switches.TruncateScaledDecimal = truncate; + SqlDecimal adjusted = TdsParser.AdjustDecimalScale( + decimal.Parse(value, CultureInfo.InvariantCulture), scale); + + Assert.Equal(truncate ? truncated : rounded, adjusted.ToString()); + Assert.Equal(scale, adjusted.Scale); + Assert.Equal(precision, adjusted.Precision); + } + } + + /// + /// Rescaling the CLR decimal limits must not reintroduce the overflow fixed by PR #4443. + /// + [Theory] + [InlineData(2)] + [InlineData(9)] + public void AdjustDecimalScale_LargeValuesRemainSqlDecimal(int scale) + { + foreach (decimal value in new[] { decimal.MinValue, decimal.MaxValue }) + { + SqlDecimal adjusted = TdsParser.AdjustDecimalScale(value, scale); + + Assert.Equal(value.ToString(CultureInfo.InvariantCulture) + "." + new string('0', scale), adjusted.ToString()); + Assert.Equal(scale, adjusted.Scale); + Assert.Equal(29 + scale, adjusted.Precision); + Assert.Throws(() => adjusted.Value); + } + } + + /// + /// Serializes one RPC parameter into a fresh parser buffer without connecting to a server. + /// + /// The CLR decimal or SqlDecimal parameter value. + /// The declared parameter precision. + /// The declared parameter scale. + private static void WriteDecimalParameter(object value, byte precision, byte scale) + { + const BindingFlags Flags = BindingFlags.Instance | BindingFlags.NonPublic; + TdsParser parser = new(false, false); + object state = typeof(TdsParser).GetField("_physicalStateObj", Flags)!.GetValue(parser)!; + SqlParameter parameter = new("@Value", SqlDbType.Decimal) + { + Value = value, + Precision = precision, + Scale = scale + }; + parameter.Validate(0, false); + + using SqlCommand command = new(); + MethodInfo write = typeof(TdsParser).GetMethod("TDSExecuteRPCAddParameter", Flags)!; + Assert.Null(write.Invoke(parser, new object[] + { + state, parameter, parameter.InternalMetaType, (byte)0, command, false + })); + } + } +} From 20ada1dc59bacd53e3bca3747f40dc98315a9550 Mon Sep 17 00:00:00 2001 From: Saurabh Singh Date: Tue, 22 Sep 2026 12:58:49 -0700 Subject: [PATCH 47/51] Fix precision of rescaled zero decimal parameters (#4721) From b9debe291e72faf2870732d8601fe2db6559b051 Mon Sep 17 00:00:00 2001 From: Jeff Sharp Date: Tue, 22 Sep 2026 16:37:40 -0500 Subject: [PATCH 48/51] Use -NoProfile to avoid interference from user's profile. (#4702) --- src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj index 43aca3f1c1..fad5d71d49 100644 --- a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj +++ b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj @@ -74,6 +74,7 @@ dotnet tool run pwsh -- -NonInteractive + -NoProfile -ExecutionPolicy Unrestricted -Command "$(RepoRoot)tools\intellisense\TrimDocs.ps1 -inputFile '$(DocumentationFile)' -outputFile '$(DocumentationFile)'" From 7da9d39d5c37ae16b1948877ae118e660a01bc5e Mon Sep 17 00:00:00 2001 From: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Date: Wed, 23 Sep 2026 11:06:09 -0700 Subject: [PATCH 49/51] Fix token expiry eviction in connection pool V2 (#4734) Validate access token expiry before general checkout, preserving return and transaction-affinity behavior. Cover callback cache refresh and physical reuse across both pool implementations and sync/async opens. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SqlConnection.xml | 11 +- .../ConnectionPool/ChannelDbConnectionPool.cs | 20 +- .../AADFedAuthTokenRefreshTest.cs | 107 ++++ .../DbConnectionPoolAccessTokenTest.cs | 484 ++++++++++++++++++ 4 files changed, 616 insertions(+), 6 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/DbConnectionPoolAccessTokenTest.cs diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml index 7a9af8b6b4..d663d13a36 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml @@ -361,9 +361,14 @@ The following example creates a and a are the same, they will be grouped into the same connection pool. - When using a token callback function, the connection manages - refreshing the tokens returned by the callback. The application is - not responsible for knowing when tokens expire. + The driver manages token refresh and discards pooled connections with + expired or nearly expired tokens before reuse. Connections in use or + reused within the same active transaction are unaffected. + + + New physical connections invoke the callback only when a cached token + is missing or needs refreshing. Reusing a pooled connection does not + invoke the callback. This property is mutually exclusive with the diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs index 9017f3f920..6e60f6cdbb 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs @@ -704,7 +704,9 @@ private void PutConnectionInIdleChannel(DbConnectionInternal connection, bool pr connection.SetReturnedTime(_timeProvider.GetUtcNow().UtcDateTime); } - if (!IsLiveConnection(connection, probeLiveness)) + // Match V1: check token expiry on general checkout, not return. An expired connection + // may remain idle, but will be discarded before it can be reused outside its transaction. + if (!IsLiveConnection(connection, probeLiveness, checkAccessTokenExpiry: false)) { RemoveConnection(connection); return; @@ -1264,9 +1266,21 @@ _connectionCreationRateLimiter is not null && /// Whether to poll the physical connection to confirm it is still alive. Pass false when /// running on a thread that must not block; the remaining checks are all cheap and local. /// + /// + /// Validate the token before general checkout, but not when returning a connection to the pool. + /// /// Returns true if the connection is live and unexpired, otherwise returns false. - private bool IsLiveConnection(DbConnectionInternal connection, bool probeLiveness = true) + private bool IsLiveConnection(DbConnectionInternal connection, bool probeLiveness = true, bool checkAccessTokenExpiry = true) { + if (checkAccessTokenExpiry && connection.IsAccessTokenExpired) + { + SqlClientEventSource.Log.TryPoolerTraceEvent( + "ChannelDbConnectionPool.IsLiveConnection | INFO | {0}, Connection {1}, will not be reused because its access token has expired or is about to expire.", + Id, + connection.ObjectID); + return false; + } + // Connection has been sitting idle longer than the configured idle timeout. // Checked before the (potentially expensive) liveness probe so an idle-expired // connection is discarded without an SNI round-trip. @@ -1548,7 +1562,7 @@ private async Task GetInternalConnection( { // Skip the liveness/idle/generation gate at the bottom of the loop: // GetFromTransactedPool has already probed liveness, and a transacted - // connection is exempt from idle-timeout, load-balance and + // connection is exempt from token-expiry, idle-timeout, load-balance and // clear-generation eviction because closing it would abort its // (possibly distributed) transaction. break; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AADFedAuthTokenRefreshTest/AADFedAuthTokenRefreshTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AADFedAuthTokenRefreshTest/AADFedAuthTokenRefreshTest.cs index e5616776b1..c79b2edcb7 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AADFedAuthTokenRefreshTest/AADFedAuthTokenRefreshTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AADFedAuthTokenRefreshTest/AADFedAuthTokenRefreshTest.cs @@ -4,7 +4,13 @@ using System; using System.Diagnostics; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; using Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.Common.SystemDataInternals; +using Microsoft.Data.SqlClient.ManualTesting.Tests.SystemDataInternals; +using Microsoft.Data.SqlClient.Tests.Common; using Xunit; using Xunit.Abstractions; @@ -87,6 +93,107 @@ public void FedAuthTokenRefreshTest() } } + /// + /// Verifies both pools replace connections with expired or nearly expired tokens and + /// invoke the callback when the cached token also needs refreshing, for Open and OpenAsync. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsAADPasswordConnStrSetup))] + [InlineData(false, false, -1)] + [InlineData(false, true, -1)] + [InlineData(true, false, -1)] + [InlineData(true, true, -1)] + [InlineData(false, false, 5)] + [InlineData(false, true, 5)] + [InlineData(true, false, 5)] + [InlineData(true, true, 5)] + public async Task AccessTokenCallback_PooledConnectionIsReplacedOnExpiry(bool usePoolV2, bool async, int expiresInSeconds) + { + using var poolVersion = new ConnectionPoolVersionScope(usePoolV2); + string[] credentialKeys = { "Authentication", "User ID", "Password", "UID", "PWD" }; + var builder = new SqlConnectionStringBuilder( + DataTestUtility.RemoveKeysInConnStr(DataTestUtility.AADPasswordConnectionString, credentialKeys)) + { + Pooling = true, + MinPoolSize = 0, + MaxPoolSize = 1, + ConnectTimeout = 30, + Enlist = false + }; + var credential = DataTestUtility.GetTokenCredential(); + SqlAuthenticationToken callbackToken = null; + int callbackInvocations = 0; + using var connection = new SqlConnection(builder.ConnectionString) + { + AccessTokenCallback = async (parameters, cancellationToken) => + { + Interlocked.Increment(ref callbackInvocations); + const string suffix = "/.default"; + string scope = parameters.Resource.EndsWith(suffix) ? parameters.Resource : parameters.Resource + suffix; + AccessToken token = await credential.GetTokenAsync(new TokenRequestContext(new[] { scope }), cancellationToken); + callbackToken = new SqlAuthenticationToken(token.Token, token.ExpiresOn); + return callbackToken; + } + }; + + Task OpenConnection() + { + if (async) + { + return connection.OpenAsync(); + } + connection.Open(); + return Task.CompletedTask; + } + + // The empty pool requires a physical login, which invokes the callback and caches its token. + await OpenConnection(); + object original = connection.GetInternalConnection(); + Assert.NotNull(callbackToken); + int callbackCountAfterLogin = callbackInvocations; + Assert.True(callbackCountAfterLogin > 0); + // Close returns the physical connection to the pool; reopening reuses it without authentication. + connection.Close(); + await OpenConnection(); + Assert.Same(original, connection.GetInternalConnection()); + Assert.Equal(callbackCountAfterLogin, callbackInvocations); + + // The connection's expiry controls eviction; the cached token's expiry controls callback refresh. + // Age both without changing the real token or waiting. Five seconds is within the 30-second + // checkout buffer; minus one second covers an already expired token. + DateTimeOffset expiry = DateTimeOffset.UtcNow.AddSeconds(expiresInSeconds); + object cachedContext = FedAuthTokenHelper.GetAuthenticationContextValue(connection); + FieldInfo cacheExpiryField = cachedContext.GetType().GetField("_expirationTime", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(cacheExpiryField); + cacheExpiryField.SetValue(cachedContext, expiry.UtcDateTime); + FieldInfo tokenField = original.GetType().GetField("_fedAuthToken", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(tokenField); + object expiringToken = Activator.CreateInstance(tokenField.FieldType, + BindingFlags.Instance | BindingFlags.NonPublic, binder: null, + args: new object[] { new SqlAuthenticationToken(callbackToken.AccessToken, expiry) }, + culture: null); + tokenField.SetValue(original, expiringToken); + // Expiry is checked on checkout, not return, so Close does not invoke the callback. + connection.Close(); + Assert.Equal(callbackCountAfterLogin, callbackInvocations); + + // Checkout discards the expired physical connection rather than reauthenticating it. + // Its replacement needs a login, and the expiring cache entry forces another callback. + // The credential may still return the same valid token; callback invocation is what matters. + await OpenConnection(); + Assert.NotSame(original, connection.GetInternalConnection()); + Assert.True(callbackInvocations > callbackCountAfterLogin); + object replacement = connection.GetInternalConnection(); + int callbackCountAfterRefresh = callbackInvocations; + // The replacement now has a valid token, so another reopen reuses it without another callback. + connection.Close(); + await OpenConnection(); + Assert.Same(replacement, connection.GetInternalConnection()); + Assert.Equal(callbackCountAfterRefresh, callbackInvocations); + using SqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT 1"; + Assert.Equal(1, async ? await command.ExecuteScalarAsync() : command.ExecuteScalar()); + } + [Conditional("DEBUG")] private void LogInfo(string message) { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/DbConnectionPoolAccessTokenTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/DbConnectionPoolAccessTokenTest.cs new file mode 100644 index 0000000000..c6e810cdd2 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/DbConnectionPoolAccessTokenTest.cs @@ -0,0 +1,484 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using System.Transactions; +using Microsoft.Data.ProviderBase; +using Microsoft.Data.SqlClient.ConnectionPool; +using Microsoft.Data.SqlClient.Tests.Common; +using Microsoft.SqlServer.TDS.PreLogin; +using Microsoft.SqlServer.TDS.Servers; +using Xunit; + +using static Microsoft.Data.SqlClient.UnitTests.ConnectionPool.PoolTestHarness; + +namespace Microsoft.Data.SqlClient.UnitTests.ConnectionPool +{ + /// + /// Verifies checkout-time token eviction separately from pool token-cache refresh. + /// The collection isolates pool-version switches; simulated logins need no Azure credentials. + /// + [Collection(SimulatedServerTestCollection.Name)] + public class DbConnectionPoolAccessTokenTest + { + private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(30); + + /// + /// Both pools defer token validation until checkout, reusing valid connections and replacing expired ones. + /// + [Theory] + [InlineData(PoolImplementation.WaitHandle, false, false)] + [InlineData(PoolImplementation.WaitHandle, false, true)] + [InlineData(PoolImplementation.WaitHandle, true, false)] + [InlineData(PoolImplementation.WaitHandle, true, true)] + [InlineData(PoolImplementation.Channel, false, false)] + [InlineData(PoolImplementation.Channel, false, true)] + [InlineData(PoolImplementation.Channel, true, false)] + [InlineData(PoolImplementation.Channel, true, true)] + public void Checkout_ValidatesIdleAccessToken(PoolImplementation implementation, bool async, bool expired) + { + using var fixture = new TokenPool(implementation); + using var owner = new SqlConnection(); + TokenConnection original = Request(fixture.Pool, owner, async); + original.Expired = expired; + int checks = original.ExpiryChecks; + fixture.Pool.ReturnInternalConnection(original, owner); + + // Like V1, returning a connection does not evaluate its token or refresh credentials. + Assert.Equal(checks, original.ExpiryChecks); + Assert.False(original.Disposed); + Assert.Equal(1, fixture.Pool.IdleCount); + + // This hits idle checkout. With one pool slot, eviction must free capacity for its replacement. + TokenConnection served = Request(fixture.Pool, owner, async); + Assert.Equal(!expired, ReferenceEquals(original, served)); + Assert.Equal(expired, original.Disposed); + Assert.True(original.ExpiryChecks > checks); + Assert.False(served.Expired); + Assert.Equal(1, fixture.Pool.Count); + fixture.Pool.ReturnInternalConnection(served, owner); + } + + /// + /// Freshly created connections must pass the expiry gate before activation, just like idle connections. + /// + [Theory] + [InlineData(PoolImplementation.WaitHandle, false)] + [InlineData(PoolImplementation.WaitHandle, true)] + [InlineData(PoolImplementation.Channel, false)] + [InlineData(PoolImplementation.Channel, true)] + public void Checkout_RejectsNewConnectionWithExpiredToken(PoolImplementation implementation, bool async) + { + // Only the first creation is expired, so retrying can succeed without waiting for time to pass. + using var fixture = new TokenPool(implementation, expireFirstCreation: true); + using var owner = new SqlConnection(); + TokenConnection served = Request(fixture.Pool, owner, async); + + Assert.Equal(2, fixture.Factory.Created.Count); + TokenConnection expired = fixture.Factory.Created[0]; + Assert.True(expired.Disposed); + // Activation would assign the connection to the caller; expiry must be rejected before then. + Assert.Equal(0, expired.Activations); + Assert.Same(fixture.Factory.Created[1], served); + Assert.Equal(1, fixture.Pool.Count); + fixture.Pool.ReturnInternalConnection(served, owner); + } + + /// + /// A direct channel handoff must validate expiry even though it bypasses the idle fast-path check. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task WaitingCheckout_RejectsExpiredToken(bool async) + { + using var fixture = new TokenPool(PoolImplementation.Channel); + var pool = (ChannelDbConnectionPool)fixture.Pool; + using var owner = new SqlConnection(); + using var waitingOwner = new SqlConnection(); + TokenConnection original = Request(pool, owner, async); + Task pending = Task.Run(() => Request(pool, waitingOwner, async)); + try + { + // Wait until the request is reading the channel, not merely scheduled on another thread. + // Returning the expired connection then exercises the post-wait gate, not idle lookup. + Assert.True(SpinWait.SpinUntil(() => pool.Reclaimer.ParkedWaiters == 1, WaitTimeout), + "The request did not reach the idle-channel wait."); + original.Expired = true; + pool.ReturnInternalConnection(original, owner); + + Assert.Same(pending, await Task.WhenAny(pending, Task.Delay(WaitTimeout))); + TokenConnection served = await pending; + Assert.NotSame(original, served); + Assert.True(original.Disposed); + Assert.False(served.Expired); + Assert.Equal(1, pool.Count); + pool.ReturnInternalConnection(served, waitingOwner); + } + finally + { + pool.Shutdown(); + Assert.Same(pending, await Task.WhenAny(pending, Task.Delay(WaitTimeout))); + } + } + + /// + /// Transaction affinity overrides expiry eviction until completion, avoiding disruption of the active transaction. + /// + [Theory] + [InlineData(PoolImplementation.WaitHandle, false)] + [InlineData(PoolImplementation.WaitHandle, true)] + [InlineData(PoolImplementation.Channel, false)] + [InlineData(PoolImplementation.Channel, true)] + public void TransactionCheckout_PreservesExpiredConnectionUntilTransactionEnds(PoolImplementation implementation, bool async) + { + using var fixture = new TokenPool(implementation); + using var owner = new SqlConnection(); + using var transaction = new CommittableTransaction(); + TokenConnection original; + using (var scope = new TransactionScope(transaction)) + { + original = Request(fixture.Pool, owner, async); + original.Expired = true; + int checks = original.ExpiryChecks; + fixture.Pool.ReturnInternalConnection(original, owner); + + // Return parks this connection in the transacted store, which takes precedence over idle reuse. + // The same transaction must get it back without checking expiry or breaking enlistment. + TokenConnection enlisted = Request(fixture.Pool, owner, async); + Assert.Same(original, enlisted); + Assert.Equal(checks, original.ExpiryChecks); + Assert.False(original.Disposed); + fixture.Pool.ReturnInternalConnection(enlisted, owner); + scope.Complete(); + } + transaction.Commit(); + + // Completion releases the connection to general circulation, where expiry eviction applies again. + TokenConnection served = Request(fixture.Pool, owner, async); + Assert.NotSame(original, served); + Assert.True(original.Disposed); + fixture.Pool.ReturnInternalConnection(served, owner); + } + + /// + /// Physical-connection expiry triggers replacement; only an expiring cached token requires another callback. + /// + [Theory] + [InlineData(false, false, -1, false)] + [InlineData(false, true, -1, false)] + [InlineData(true, false, -1, false)] + [InlineData(true, true, -1, false)] + [InlineData(false, false, 300, false)] + [InlineData(false, true, 300, false)] + [InlineData(true, false, 300, false)] + [InlineData(true, true, 300, false)] + [InlineData(false, false, -1, true)] + [InlineData(false, true, -1, true)] + [InlineData(true, false, -1, true)] + [InlineData(true, true, -1, true)] + [InlineData(false, false, 300, true)] + [InlineData(false, true, 300, true)] + [InlineData(true, false, 300, true)] + [InlineData(true, true, 300, true)] + public async Task AccessTokenCallback_CheckoutRejectsExpiredToken(bool usePoolV2, bool async, int expiresInSeconds, bool expireCachedToken) + { + using var poolVersion = new ConnectionPoolVersionScope(usePoolV2); + using var server = new TdsServer(new TdsServerArguments + { + FedAuthRequiredPreLoginOption = TdsPreLoginFedAuthRequiredOption.FedAuthRequired + }); + server.Start(); + var builder = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + MaxPoolSize = 1, + ConnectTimeout = 600, + Enlist = false + }; + int callbackInvocations = 0; + using var connection = new SqlConnection(builder.ConnectionString) + { + AccessTokenCallback = (_, _) => + { + Interlocked.Increment(ref callbackInvocations); + return Task.FromResult(new SqlAuthenticationToken("invalid", DateTimeOffset.UtcNow.AddHours(2))); + } + }; + + try + { + // The first physical login populates the token cache; ordinary reopen only reuses the socket. + await OpenConnection(connection, async); + var original = Assert.IsType(connection.InnerConnection); + Assert.False(original.IsAccessTokenExpired); + Assert.Equal(1, callbackInvocations); + connection.Close(); + await OpenConnection(connection, async); + Assert.Same(original, connection.InnerConnection); + Assert.Equal(1, callbackInvocations); + + // Change metadata, not wall-clock time: -1 is expired and 300 is within the 600-second buffer. + // The physical connection and the pool cache hold separate expiry values. + FieldInfo? tokenField = original.GetType().GetField("_fedAuthToken", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(tokenField); + DateTimeOffset expiry = DateTimeOffset.UtcNow.AddSeconds(expiresInSeconds); + tokenField!.SetValue(original, new SqlFedAuthToken(new SqlAuthenticationToken("invalid", expiry))); + Assert.Equal(expiresInSeconds > 0, expiry > DateTimeOffset.UtcNow); + Assert.True(original.IsAccessTokenExpired); + IDbConnectionPool pool = original.Pool; + var cachedToken = Assert.Single(pool.AuthenticationContexts); + if (expireCachedToken) + { + // Leaving the cache fresh models another login already having refreshed the pool's token. + // Aging it instead requires the replacement login to invoke the callback. + pool.AuthenticationContexts[cachedToken.Key] = new DbConnectionPoolAuthenticationContext( + cachedToken.Value.AccessToken, expiry.UtcDateTime); + } + connection.Close(); + Assert.Equal(1, callbackInvocations); + + // Checkout evicts the old physical connection. Its replacement either uses the fresh cache + // or invokes the callback and updates the cache; eviction alone must not force token acquisition. + await OpenConnection(connection, async); + Assert.NotSame(original, connection.InnerConnection); + Assert.False(connection.InnerConnection.IsAccessTokenExpired); + Assert.Equal(expireCachedToken ? 2 : 1, callbackInvocations); + Assert.True(pool.AuthenticationContexts[cachedToken.Key].ExpirationTime > expiry.UtcDateTime); + + // Once replaced, reuse must neither create another physical connection nor invoke the callback. + DbConnectionInternal replacement = connection.InnerConnection; + connection.Close(); + await OpenConnection(connection, async); + Assert.Same(replacement, connection.InnerConnection); + Assert.Equal(expireCachedToken ? 2 : 1, callbackInvocations); + } + finally + { + SqlConnection.ClearPool(connection); + } + } + + /// + /// New physical logins refresh expired or nearly expired cached tokens but reuse sufficiently valid ones. + /// + [Theory] + [InlineData(false, false, -1)] + [InlineData(false, true, -1)] + [InlineData(true, false, -1)] + [InlineData(true, true, -1)] + [InlineData(false, false, 300)] + [InlineData(false, true, 300)] + [InlineData(true, false, 300)] + [InlineData(true, true, 300)] + [InlineData(false, false, 3600)] + [InlineData(false, true, 3600)] + [InlineData(true, false, 3600)] + [InlineData(true, true, 3600)] + public async Task AccessTokenCallback_NewPhysicalConnectionHonorsCachedTokenExpiry(bool usePoolV2, bool async, int expiresInSeconds) + { + using var poolVersion = new ConnectionPoolVersionScope(usePoolV2); + using var server = new TdsServer(new TdsServerArguments + { + FedAuthRequiredPreLoginOption = TdsPreLoginFedAuthRequiredOption.FedAuthRequired + }); + server.Start(); + var builder = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + MaxPoolSize = 2, + Enlist = false + }; + int callbackInvocations = 0; + using var first = new SqlConnection(builder.ConnectionString) + { + AccessTokenCallback = (_, _) => + { + int invocation = Interlocked.Increment(ref callbackInvocations); + return Task.FromResult(new SqlAuthenticationToken($"invalid-{invocation}", DateTimeOffset.UtcNow.AddHours(2))); + } + }; + // The callback delegate is part of the pool key; share it to exercise the same token cache. + using var second = new SqlConnection(builder.ConnectionString) + { + AccessTokenCallback = first.AccessTokenCallback + }; + + try + { + await OpenConnection(first, async); + Assert.Equal(1, callbackInvocations); + IDbConnectionPool pool = first.InnerConnection.Pool; + var entry = Assert.Single(pool.AuthenticationContexts); + var cachedToken = new DbConnectionPoolAuthenticationContext( + entry.Value.AccessToken, DateTime.UtcNow.AddSeconds(expiresInSeconds)); + pool.AuthenticationContexts[entry.Key] = cachedToken; + + // Keep the first connection checked out so the second must perform a physical login. + await OpenConnection(second, async); + Assert.NotSame(first.InnerConnection, second.InnerConnection); + Assert.Same(pool, second.InnerConnection.Pool); + Assert.False(second.InnerConnection.IsAccessTokenExpired); + // -1 and 300 seconds require refresh (the cache's 10-minute window); 3600 seconds + // is beyond even its 45-minute opportunistic refresh window, so it must reuse the token. + bool refreshed = expiresInSeconds <= 600; + Assert.Equal(refreshed ? 2 : 1, callbackInvocations); + DbConnectionPoolAuthenticationContext current = pool.AuthenticationContexts[entry.Key]; + if (refreshed) + { + Assert.NotSame(cachedToken, current); + Assert.NotEqual(cachedToken.AccessToken, current.AccessToken); + Assert.True(current.ExpirationTime > cachedToken.ExpirationTime); + } + else + { + Assert.Same(cachedToken, current); + } + + // Physical reuse skips authentication entirely, regardless of which cache branch ran above. + DbConnectionInternal reused = second.InnerConnection; + second.Close(); + await OpenConnection(second, async); + Assert.Same(reused, second.InnerConnection); + Assert.Equal(refreshed ? 2 : 1, callbackInvocations); + } + finally + { + SqlConnection.ClearPool(first); + } + } + + /// + /// Exercises the selected public open API, bounding asynchronous opens with cancellation. + /// + /// Connection to open against the simulated server. + /// Whether to use OpenAsync instead of Open. + /// A task completing when the connection is open. + private static async Task OpenConnection(SqlConnection connection, bool async) + { + if (async) + { + using var cancellation = new CancellationTokenSource(WaitTimeout); + await connection.OpenAsync(cancellation.Token); + } + else + { + connection.Open(); + } + } + + /// + /// Requests a stub connection directly from a pool, handling inline and deferred completion. + /// + /// Pool under test. + /// Owner passed to connection activation. + /// Whether to supply the completion source used by asynchronous opens. + /// The connection assigned to the owner. + private static TokenConnection Request(IDbConnectionPool pool, SqlConnection owner, bool async) + { + // Async opens carry the ambient transaction in AsyncState so worker threads preserve affinity. + TaskCompletionSource? completion = async + ? new TaskCompletionSource(Transaction.Current, TaskCreationOptions.RunContinuationsAsynchronously) + : null; + bool completed = pool.TryGetConnection(owner, completion, TimeoutTimer.StartNew(WaitTimeout), out DbConnectionInternal? connection); + if (!completed) + { + Assert.NotNull(completion); + Assert.True(completion!.Task.Wait(WaitTimeout), "The connection request did not complete."); + connection = completion.Task.GetAwaiter().GetResult(); + } + return Assert.IsType(connection); + } + + /// + /// Uses one pool slot and the harness's frozen clock to isolate checkout decisions from maintenance. + /// + private sealed class TokenPool : IDisposable + { + internal TokenFactory Factory { get; } + internal IDbConnectionPool Pool { get; } + + /// Creates an isolated pool backed by controllable token connections. + /// Pool implementation to exercise. + /// Whether the first created connection starts expired. + internal TokenPool(PoolImplementation implementation, bool expireFirstCreation = false) + { + Factory = new TokenFactory(expireFirstCreation); + Pool = ConstructPool(implementation, Factory, maxPoolSize: 1, creationTimeout: 30000); + } + + /// Stops maintenance and disposes idle connections and any left checked out by a failed test. + public void Dispose() + { + Pool.Shutdown(); + Pool.Clear(); + foreach (TokenConnection connection in Factory.Created) + { + if (!connection.Disposed) + { + connection.Dispose(); + } + } + } + } + + /// + /// Records physical creations and can expire the first one to force a checkout retry. + /// + private sealed class TokenFactory(bool expireFirstCreation) : SqlConnectionFactory + { + internal List Created { get; } = new(); + + /// + protected override DbConnectionInternal CreateConnection(SqlConnectionOptions options, ConnectionPoolKey poolKey, + DbConnectionPoolGroupProviderInfo poolGroupProviderInfo, IDbConnectionPool pool, DbConnection owningConnection, TimeoutTimer timeout) + { + var connection = new TokenConnection { Expired = expireFirstCreation && Created.Count == 0 }; + Created.Add(connection); + return connection; + } + } + + /// + /// Makes expiry deterministic and records whether the pool checks, activates, or disposes the connection. + /// + private sealed class TokenConnection : ChannelDbConnectionPoolTest.StubDbConnectionInternal + { + internal bool Expired { get; set; } + internal int ExpiryChecks { get; private set; } + internal int Activations { get; private set; } + internal bool Disposed { get; private set; } + + internal override bool IsAccessTokenExpired + { + get + { + ExpiryChecks++; + return Expired; + } + } + + /// + protected override void Activate(Transaction transaction) + { + Activations++; + base.Activate(transaction); + } + + /// + public override void Dispose() + { + Disposed = true; + base.Dispose(); + } + } + } +} From 3160bb20ccb15cd3f489c7dce632b940b208ed18 Mon Sep 17 00:00:00 2001 From: Malcolm Daigle Date: Wed, 23 Sep 2026 11:16:14 -0700 Subject: [PATCH 50/51] Preserve in-flight opens when clearing connection pools (#4718) * Fix OpenAsync retry after pool clear Route pending async opens on a cleared wait-handle pool back through the connection factory so they can complete from the replacement pool instead of surfacing a misleading pool timeout. Add a regression test that parks an async pending open, clears the pool group, and verifies completion from a replacement pool. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Preserve in-flight opens when clearing wait-handle pools Remove shutdown interruption and retry routing. Let admitted requests finish on the retired pool and dispose their connections on return. Preserve error expiry for remaining waiters. Cover sync and async clearing, cancellation, timeouts, and error expiry. Add a SQL Server regression workload for #4714. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Make pool-clear regression coverage deterministic Remove the concurrent stress workload and elapsed-time assertions. Gate physical creation and assert exact request ordering and creation counts around pool shutdown. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Remove pool-clearing documentation changes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Leave retired pool clearing to the factory Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Clarify gated pool shutdown test scenario Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Separate pool-clear scenario from test plumbing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Organize pool regression tests as arrange act assert Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Explain pool regression test interleavings inline Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Trim blocking-period test commentary Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Explain regression conditions and assertion evidence Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Clarify shutdown test walkthrough Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Match pool-clear comments to shutdown walkthrough Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../WaitHandleDbConnectionPool.cs | 70 +----- ...andleDbConnectionPoolBlockingPeriodTest.cs | 28 +++ .../WaitHandleDbConnectionPoolShutdownTest.cs | 234 +++++++++++++----- .../PoolClearDuringOpenTests.cs | 205 +++++++++++++++ 4 files changed, 413 insertions(+), 124 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/PoolClearDuringOpenTests.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs index afaa827fce..887bcd5c35 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs @@ -900,12 +900,8 @@ public bool TryGetConnection(DbConnection owningObject, TaskCompletionSource {0}, Pool is shutting down; abandoning wait.", Id); - if (waitResult == SEMAPHORE_HANDLE || waitResult == WAIT_ABANDONED + SEMAPHORE_HANDLE) - { - try - { - _waitHandles.PoolSemaphore.Release(1); - } - catch (SemaphoreFullException) - { - // Pool semaphore was already saturated by Shutdown's bulk release; safe to ignore. - } - } - Interlocked.Decrement(ref _waitCount); - connection = null; - return false; - } - // From the WaitAny docs: "If more than one object became signaled during // the call, this is the array index of the signaled object with the // smallest index value of all the signaled objects." This is important @@ -1597,37 +1565,17 @@ public void Shutdown() } State = ShuttingDown; - // Dispose all background timers so they no longer schedule new work. - // Note that any timer callback already in flight may still observe State == ShuttingDown - // and short-circuit (see CleanupCallback / ErrorCallback). + // Stop maintenance, but let admitted requests finish. Their connections are + // destroyed by DeactivateObject when returned to this retired pool. Timer cleanup = Interlocked.Exchange(ref _cleanupTimer, null); cleanup?.Dispose(); - _errorState.Dispose(); - - // Wake any threads parked in WaitHandle.WaitAny by releasing as many semaphore - // slots as there are recorded waiters. Using _waitCount (rather than MaxPoolSize) - // avoids ArgumentOutOfRangeException when MaxPoolSize == 0 (unlimited) and ensures - // we wake every parked waiter even when _waitCount exceeds MaxPoolSize. Waiters - // observe State is not Running after wake-up and bail. - int waitersToWake = Volatile.Read(ref _waitCount); - if (waitersToWake > 0) - { - try - { - _waitHandles.PoolSemaphore.Release(waitersToWake); - } - catch (SemaphoreFullException) - { - // Semaphore already saturated; nothing to do. - } - } + // Keep the cached error and its expiry timer available to admitted waiters. + // Disposing the error state here leaves ErrorEvent signaled without an error. - // Reuse Clear() to doom every connection (including active checked-out ones), drain - // both idle stacks, and reclaim emancipated objects. Active connections destroy - // themselves on return either via the doom flag or via DeactivateObject's - // State == ShuttingDown branch. - Clear(); + // Leave Clear() to the factory's explicit-clear or deferred-pruning path. + // Shutdown can run under the pool-group lock, where reclamation and connection + // disposal must not be added before the pool is queued for release. } // TransactionEnded merely provides the plumbing for DbConnectionInternal to access the transacted pool diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs index c5d628c618..163023a0f4 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs @@ -130,6 +130,34 @@ public void TryGetConnection_WhenFactoryThrows_EntersBlockingPeriod() Assert.Equal(1, factory.CreateConnectionCallCount); } + /// + /// Shutdown preserves the cached error for admitted waiters and lets its timer expire. + /// + [Fact] + public void Shutdown_WhileBlocked_PreservesErrorUntilExpiry() + { + // Arrange + var clock = new FakeTimeProvider(); + SqlException failure = SqlExceptionHelper.CreateSqlException("server unreachable"); + var factory = new ConfigurableSqlConnectionFactory(_ => throw failure); + var pool = CreatePool(factory, timeProvider: clock); + using var owner = new SqlConnection(); + + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + + // Act + pool.Shutdown(); + + // Assert + Assert.True(pool.ErrorOccurred); + + // Act: expire the preserved blocking period. + clock.Advance(TimeSpan.FromSeconds(5)); + + // Assert + Assert.False(pool.ErrorOccurred); + } + /// /// Verifies that once the pool is in the blocking period, a subsequent request fast-fails /// with the cached exception without invoking the connection factory again. The first throw diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolShutdownTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolShutdownTest.cs index 17ed3e63f8..c0be93f156 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolShutdownTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolShutdownTest.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Data.Common; using System.Threading; using System.Threading.Tasks; using Microsoft.Data.Common.ConnectionString; @@ -17,7 +18,7 @@ namespace Microsoft.Data.SqlClient.UnitTests.ConnectionPool /// public class WaitHandleDbConnectionPoolShutdownTest { - private static WaitHandleDbConnectionPool CreatePool(int maxPoolSize = 5) + private static WaitHandleDbConnectionPool CreatePool(int maxPoolSize = 5, SqlConnectionFactory? factory = null) { var poolGroupOptions = new DbConnectionPoolGroupOptions( poolByIdentity: false, @@ -34,7 +35,7 @@ private static WaitHandleDbConnectionPool CreatePool(int maxPoolSize = 5) poolGroupOptions); var pool = new WaitHandleDbConnectionPool( - new WaitHandleDbConnectionPoolTransactionTest.MockSqlConnectionFactory(), + factory ?? new WaitHandleDbConnectionPoolTransactionTest.MockSqlConnectionFactory(), dbConnectionPoolGroup, DbConnectionPoolIdentity.NoIdentity, new DbConnectionPoolProviderInfo()); @@ -67,13 +68,17 @@ public void Shutdown_DisposesCleanupTimer() Assert.Null(pool._cleanupTimer); } - // Drains idle stacks. + /// + /// Leaves idle connections for the factory's explicit or deferred Clear call. + /// [Fact] - public void Shutdown_DrainsIdleStacks() + public void Shutdown_LeavesIdleConnectionsUntilClear() { + // Arrange var pool = CreatePool(); - // Vend a few connections then return them so they sit in _stackNew. + // An empty pool would pass even if Shutdown still called Clear internally. + // Keep idle inventory so the two lifecycle operations have distinguishable effects. var owner1 = new SqlConnection(); var owner2 = new SqlConnection(); pool.TryGetConnection(owner1, taskCompletionSource: null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? c1); @@ -86,10 +91,32 @@ public void Shutdown_DrainsIdleStacks() Assert.Equal(2, pool.IdleCount); Assert.Equal(2, pool.Count); - pool.Shutdown(); - - Assert.Equal(0, pool.IdleCount); - Assert.Equal(0, pool.Count); + try + { + // Act + pool.Shutdown(); + + // Assert: unchanged inventory and poolability detect both effects of an + // unintended Clear: draining idle objects and marking them non-poolable. + Assert.False(pool.IsRunning); + Assert.Equal(2, pool.IdleCount); + Assert.Equal(2, pool.Count); + Assert.True(c1!.CanBePooled); + Assert.True(c2!.CanBePooled); + + // Act: perform the Clear that the factory owns, separately from shutdown. + pool.Clear(); + + // Assert + Assert.Equal(0, pool.IdleCount); + Assert.Equal(0, pool.Count); + } + finally + { + // Cleanup + pool.Shutdown(); + pool.Clear(); + } } // Shutdown is idempotent. @@ -160,70 +187,151 @@ public void TryGetConnection_Async_AfterShutdown_ShortCircuits_NoPendingOpenSche Assert.Equal(0, Volatile.Read(ref pool._waitCount)); } - // Shutdown wakes up a thread parked in WaitHandle.WaitAny. - [Trait("category", "flaky")] - // Failed Microsoft.Data.SqlClient.UnitTests.ConnectionPool.WaitHandleDbConnectionPoolShutdownTest.Shutdown_UnblocksSyncWaiter [5 s] - // ##[error]EXEC(0,0): Error Message: - // EXEC : error Message: [D:\a\_work\1\s\build.proj] - // Waiter did not park within 5s. - // Stack Trace: - // at Microsoft.Data.SqlClient.UnitTests.ConnectionPool.WaitHandleDbConnectionPoolShutdownTest.Shutdown_UnblocksSyncWaiter() in D:\a\_work\1\s\src\Microsoft.Data.SqlClient\tests\UnitTests\ConnectionPool\WaitHandleDbConnectionPoolShutdownTest.cs:line 207 - [Fact] - public void Shutdown_UnblocksSyncWaiter() + /// + /// Starts with no idle connections. Pauses the first physical connection creation + /// while it holds the creation semaphore, then admits a second request that waits + /// for that semaphore. Shuts down the pool before releasing the first creation. + /// Both requests create their own connections on the retired pool, and both + /// connections are destroyed when returned. In the cancellation case, the worker + /// returns and destroys the second connection instead of delivering it to the caller. + /// + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Shutdown_InFlightRequest_CompletesOnRetiredPool(bool async, bool cancel) { - var pool = CreatePool(maxPoolSize: 1); + // Arrange + // Initiate a request to the pool. It will be blocked by the gated connection factory. + using var factory = new GatedConnectionFactory(); + var pool = CreatePool(maxPoolSize: 2, factory: factory); + using var firstOwner = new SqlConnection(); + using var pendingOwner = new SqlConnection(); + var completion = new TaskCompletionSource(); + Task first = Acquire(pool, firstOwner, completion: null); + Task? pending = null; + + // Make sure the first request is blocked in the connection factory. Then, initiate a second request. + // The second request will be blocked on the create semaphore in the pool. + Assert.True(factory.Entered.Wait(TimeSpan.FromSeconds(10))); + pending = Acquire(pool, pendingOwner, async ? completion : null); + // Make sure the second request is also blocked. + Assert.True(SpinWait.SpinUntil(() => Volatile.Read(ref pool._waitCount) == 2, TimeSpan.FromSeconds(10))); + Assert.Equal(1, factory.CreateCount); + Assert.False(first.IsCompleted); + Assert.False(pending.IsCompleted); + + // Act + // Now, shut down the pool. New requests will no longer be accepted, but in-flight requests should proceed. + pool.Shutdown(); + if (cancel) + { + completion.SetCanceled(); + } - // Saturate the pool. - var owner = new SqlConnection(); - Assert.True(pool.TryGetConnection(owner, taskCompletionSource: null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? blocking)); - Assert.NotNull(blocking); + // Unblock the first request, allowing both to proceed, in turn. + factory.Release.Set(); - // Park a sync waiter on a worker thread with a long creation timeout. - DbConnectionInternal? waiterResult = null; - bool waiterCompleted = false; - Exception? waiterEx = null; + // Assert + // Wait for the first request to complete + Assert.False(pool.IsRunning); + Assert.Same(first, await Task.WhenAny(first, Task.Delay(TimeSpan.FromSeconds(10)))); + DbConnectionInternal? firstConnection = await first; + Assert.NotNull(firstConnection); + Assert.Same(pool, firstConnection.Pool); + + // Wait for the second request to complete + Assert.Same(pending, await Task.WhenAny(pending, Task.Delay(TimeSpan.FromSeconds(10)))); + if (cancel) + { + await Assert.ThrowsAnyAsync(() => pending); + Assert.True(SpinWait.SpinUntil(() => pool.Count == 1 && Volatile.Read(ref pool._waitCount) == 0, TimeSpan.FromSeconds(10))); + } + else + { + DbConnectionInternal? pendingConnection = await pending; + Assert.NotNull(pendingConnection); + Assert.Same(pool, pendingConnection.Pool); + Assert.Equal(0, Volatile.Read(ref pool._waitCount)); + } + + // Assert that both requests created new connections + Assert.Equal(2, factory.CreateCount); - var t = new Thread(() => + // Cleanup + factory.Release.Set(); + pool.Shutdown(); + await ReturnWhenCompleted(pool, firstOwner, first); + if (pending is not null) { - try - { - waiterCompleted = pool.TryGetConnection( - new SqlConnection(), - taskCompletionSource: null, - TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), - out waiterResult); - } - catch (Exception ex) + await ReturnWhenCompleted(pool, pendingOwner, pending); + } + + // Assert: returned connections were destroyed rather than pooled. + Assert.Equal(0, pool.IdleCount); + Assert.Equal(0, pool.Count); + } + + /// Starts a sync acquisition on a dedicated thread or queues an async acquisition. + private static Task Acquire(WaitHandleDbConnectionPool pool, SqlConnection owner, + TaskCompletionSource? completion) + { + TimeoutTimer timer = TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)); + if (completion is null) + { + return Task.Factory.StartNew(() => { - waiterEx = ex; - } - }) - { IsBackground = true }; - t.Start(); - - // Wait deterministically until the worker has incremented _waitCount, which - // happens immediately before it enters WaitHandle.WaitAny. Polling avoids the - // CI-flakiness of a fixed Thread.Sleep on slow agents. Volatile.Read ensures - // we see the worker's Interlocked.Increment without depending on CPU memory - // ordering of plain int reads. - var deadline = DateTime.UtcNow.AddSeconds(5); - while (DateTime.UtcNow < deadline && Volatile.Read(ref pool._waitCount) < 1) + Assert.True(pool.TryGetConnection(owner, null, timer, out DbConnectionInternal connection)); + return (DbConnectionInternal?)connection; + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + } + // A false return here means async acquisition was queued, not that it failed. + // Its eventual result is delivered through completion.Task. + Assert.False(pool.TryGetConnection(owner, completion, timer, out DbConnectionInternal pending)); + Assert.Null(pending); + return completion.Task!; + } + + /// Drains test work and returns successful acquisitions even when an assertion failed. + private static async Task ReturnWhenCompleted(WaitHandleDbConnectionPool pool, SqlConnection owner, Task task) + { + Assert.Same(task, await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(20)))); + if (task.Status == TaskStatus.RanToCompletion && task.Result is { } connection) + { + pool.ReturnInternalConnection(connection, owner); + } + else if (task.IsFaulted) { - Thread.Yield(); + // Observe failures during assertion cleanup without replacing the original failure. + _ = task.Exception; } - Assert.True(Volatile.Read(ref pool._waitCount) >= 1, "Waiter did not park within 5s."); - Assert.True(t.IsAlive, "Waiter should be parked, but thread already exited."); + } - pool.Shutdown(); + /// Holds the first physical creation so a second acquisition waits on its semaphore. + private sealed class GatedConnectionFactory : WaitHandleDbConnectionPoolTransactionTest.MockSqlConnectionFactory, IDisposable + { + internal readonly ManualResetEventSlim Entered = new(); + internal readonly ManualResetEventSlim Release = new(); + private int _calls; - Assert.True(t.Join(TimeSpan.FromSeconds(5)), "Waiter did not unblock within 5s of Shutdown."); - // Acceptable outcomes: either returned false/null (timed out / abandoned) or - // returned true/null (state-check short-circuit). Either way, it must NOT block - // forever, and it must NOT vend a real connection from a shut-down pool. - Assert.Null(waiterResult); - Assert.Null(waiterEx); - // Suppress unused warning - presence of waiterCompleted just documents the contract. - _ = waiterCompleted; + internal int CreateCount => Volatile.Read(ref _calls); + + protected override DbConnectionInternal CreateConnection(SqlConnectionOptions options, ConnectionPoolKey poolKey, + DbConnectionPoolGroupProviderInfo poolGroupProviderInfo, IDbConnectionPool pool, DbConnection owningConnection, TimeoutTimer timeout) + { + if (Interlocked.Increment(ref _calls) == 1) + { + Entered.Set(); + Assert.True(Release.Wait(TimeSpan.FromSeconds(15)), "Physical creation was not released."); + } + return base.CreateConnection(options, poolKey, poolGroupProviderInfo, pool, owningConnection, timeout); + } + + public void Dispose() + { + Entered.Dispose(); + Release.Dispose(); + } } // Startup() must be a no-op when the pool has already been shut down. Without the diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/PoolClearDuringOpenTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/PoolClearDuringOpenTests.cs new file mode 100644 index 0000000000..9b6bf1f2b4 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/PoolClearDuringOpenTests.cs @@ -0,0 +1,205 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient.ConnectionPool; +using Microsoft.Data.SqlClient.Tests.Common; +using Microsoft.SqlServer.TDS.Servers; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests; + +/// +/// Exercises public pool clearing while one login is in flight and another open is waiting. +/// +[Collection(SimulatedServerTestCollection.Name)] +public class PoolClearDuringOpenTests +{ + /// + /// Blocks the first login, admits a second open, then clears the pool before releasing + /// the login. Both opens finish on the retired pool and their connections are destroyed + /// on close. A subsequent open uses a replacement pool. + /// + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ClearDuringOpen_InFlightConnectionsAreDestroyedOnClose(bool async, bool clearAll) + { + // Arrange + // Initiate an open using the wait-handle pool. Its login will be blocked by the gated server. + using var switches = new LocalAppContextSwitchesHelper { UseConnectionPoolV2 = false }; + using var server = new GatedLoginServer(); + using var first = new SqlConnection(server.ConnectionString); + using var pending = new SqlConnection(server.ConnectionString); + using var replacement = new SqlConnection(server.ConnectionString); + Task opens = Open(first, async); + + try + { + // Make sure the first open is blocked during login. Then, initiate a second open. + // The second open must wait for the first creation to finish. + server.WaitForFirstLogin(); + var retiredPool = Assert.IsType(first.PoolGroup.GetConnectionPool(SqlConnectionFactory.Instance)); + Assert.False(opens.IsCompleted); + Task pendingOpen = Open(pending, async); + opens = Task.WhenAll(opens, pendingOpen); + + // Make sure the second open is pending on the same pool and has not started its login. + AssertPendingOpen(pending, pendingOpen, retiredPool, async); + Assert.Same(first.PoolGroup, pending.PoolGroup); + Assert.Equal(1, server.LoginCount); + + // Act + // Now, clear the pool. New opens should use a new pool, but these in-flight opens should proceed. + if (clearAll) + { + SqlConnection.ClearAllPools(); + } + else + { + SqlConnection.ClearPool(first); + } + + // Unblock the first login, allowing both opens to proceed, in turn. + server.ReleaseFirstLogin(); + + // Assert + // Wait for both opens to complete, then confirm they succeeded on the original pool. + await AssertCompletes(opens); + await opens; + + Assert.False(retiredPool.IsRunning); + Assert.Equal(ConnectionState.Open, first.State); + Assert.Equal(ConnectionState.Open, pending.State); + Assert.Same(retiredPool, first.InnerConnection.Pool); + Assert.Same(retiredPool, pending.InnerConnection.Pool); + + // Assert that both opens created new connections. + Assert.Equal(2, retiredPool.Count); + Assert.Equal(2, server.LoginCount); + + // Act + // Initiate another open after clearing. This one should use a new pool. + await Open(replacement, async); + + // Assert + // Make sure the new open used a different, running pool and performed another login. + Assert.NotSame(retiredPool, replacement.InnerConnection.Pool); + Assert.True(replacement.InnerConnection.Pool.IsRunning); + Assert.Equal(3, server.LoginCount); + + // Act + // Close the connections that completed on the retired pool. They should be destroyed, not reused. + first.Close(); + pending.Close(); + + // Assert: the retired pool has no connections or pending requests left. + Assert.Equal(0, retiredPool.Count); + Assert.Equal(0, retiredPool.IdleCount); + Assert.Equal(0, Volatile.Read(ref retiredPool._waitCount)); + } + finally + { + // Cleanup + server.ReleaseFirstLogin(); + await AssertCompletes(opens); + if (opens.IsFaulted) + { + // Observe background failures without replacing an earlier assertion failure. + _ = opens.Exception; + } + first.Close(); + pending.Close(); + replacement.Close(); + SqlConnection.ClearPool(first); + } + } + + /// Confirms the second open is admitted before the test clears its pool. + private static void AssertPendingOpen(SqlConnection pending, Task open, WaitHandleDbConnectionPool pool, bool async) + { + if (async) + { + // The second async open is queued behind the worker handling the first login. + // It has not entered the pool's wait loop yet, so _waitCount will still be 1. + Assert.Equal(ConnectionState.Connecting, pending.State); + } + else + { + // The sync opens run on separate threads. Wait until the second has entered + // the pool and is blocked on the creation semaphore before allowing the test to clear it. + Assert.True(SpinWait.SpinUntil(() => Volatile.Read(ref pool._waitCount) == 2, TimeSpan.FromSeconds(10)), + "Second open did not wait for the creation semaphore."); + } + Assert.False(open.IsCompleted); + } + + /// + /// Prevents a stuck open from hanging the test. The caller awaits the completed task + /// separately to check whether the opens succeeded. + /// + private static async Task AssertCompletes(Task task) => + Assert.Same(task, await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(20)))); + + /// Runs synchronous opens on a dedicated thread so the test can release the login. + private static Task Open(SqlConnection connection, bool async) => + async + ? connection.OpenAsync() + : Task.Factory.StartNew(connection.Open, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + /// Owns the simulated server and a gate that blocks only its first login. + private sealed class GatedLoginServer : IDisposable + { + private readonly TdsServer _server = new(); + private readonly ManualResetEventSlim _loginEntered = new(); + private readonly ManualResetEventSlim _releaseLogin = new(); + private int _logins; + + internal int LoginCount => Volatile.Read(ref _logins); + internal string ConnectionString { get; } + + /// Starts an isolated server with room for both admitted connections. + internal GatedLoginServer() + { + _server.OnLogin7Validated = _ => + { + if (Interlocked.Increment(ref _logins) == 1) + { + _loginEntered.Set(); + Assert.True(_releaseLogin.Wait(TimeSpan.FromSeconds(15)), "Login was not released."); + } + }; + _server.Start(); + ConnectionString = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{_server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + Pooling = true, + MaxPoolSize = 2, + ConnectTimeout = 15, + }.ConnectionString; + } + + /// Waits until the first login reaches the gate. + internal void WaitForFirstLogin() => + Assert.True(_loginEntered.Wait(TimeSpan.FromSeconds(10)), "First login did not start."); + + /// Allows the first login to finish, including during failure cleanup. + internal void ReleaseFirstLogin() => _releaseLogin.Set(); + + /// Releases the gate and disposes the server before its synchronization objects. + public void Dispose() + { + ReleaseFirstLogin(); + _server.Dispose(); + _loginEntered.Dispose(); + _releaseLogin.Dispose(); + } + } +} From d189eeb7cfafcbe6ac2bb0c63eb5cee1cfd71495 Mon Sep 17 00:00:00 2001 From: Charles Roddie Date: Wed, 23 Sep 2026 19:21:27 +0100 Subject: [PATCH 51/51] Remove DiagnosticSource.Write trim warnings in SqlDiagnosticListener (#4688) Route the 15 diagnostic events through one WriteEvent helper that carries the PublicProperties annotation on T required by Write and suppresses IL2026 once. SqlClient does no reflection when writing events; subscribers that reflect over payloads are responsible for their own reflection. Co-authored-by: Claude Opus 5 (1M context) --- .../Diagnostics/SqlDiagnosticListener.cs | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Diagnostics/SqlDiagnosticListener.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Diagnostics/SqlDiagnosticListener.cs index 40e4bb63d9..0777a527b3 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Diagnostics/SqlDiagnosticListener.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Diagnostics/SqlDiagnosticListener.cs @@ -5,6 +5,7 @@ using System; using System.Data; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; #if NET using System.Reflection; @@ -77,7 +78,7 @@ public void WriteCommandAfter( return; } - Write( + WriteEvent( SqlClientCommandAfter.Name, new SqlClientCommandAfter( operationId, @@ -104,7 +105,7 @@ public Guid WriteCommandBefore( Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientCommandBefore.Name, new SqlClientCommandBefore( operationId, @@ -132,7 +133,7 @@ public void WriteCommandError( return; } - Write( + WriteEvent( SqlClientCommandError.Name, new SqlClientCommandError ( @@ -159,7 +160,7 @@ public void WriteConnectionCloseAfter( return; } - Write( + WriteEvent( SqlClientConnectionCloseAfter.Name, new SqlClientConnectionCloseAfter ( @@ -182,7 +183,7 @@ public Guid WriteConnectionCloseBefore(SqlConnection sqlConnection, [CallerMembe Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientConnectionCloseBefore.Name, new SqlClientConnectionCloseBefore ( @@ -211,7 +212,7 @@ public void WriteConnectionCloseError( return; } - Write( + WriteEvent( SqlClientConnectionCloseError.Name, new SqlClientConnectionCloseError ( @@ -237,7 +238,7 @@ public void WriteConnectionOpenAfter( return; } - Write( + WriteEvent( SqlClientConnectionOpenAfter.Name, new SqlClientConnectionOpenAfter ( @@ -261,7 +262,7 @@ public Guid WriteConnectionOpenBefore(SqlConnection sqlConnection, [CallerMember Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientConnectionOpenBefore.Name, new SqlClientConnectionOpenBefore ( @@ -288,7 +289,7 @@ public void WriteConnectionOpenError( return; } - Write( + WriteEvent( SqlClientConnectionOpenError.Name, new SqlClientConnectionOpenError ( @@ -316,7 +317,7 @@ public void WriteTransactionCommitAfter( return; } - Write( + WriteEvent( SqlClientTransactionCommitAfter.Name, new SqlClientTransactionCommitAfter ( @@ -344,7 +345,7 @@ public Guid WriteTransactionCommitBefore( Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientTransactionCommitBefore.Name, new SqlClientTransactionCommitBefore ( @@ -374,7 +375,7 @@ public void WriteTransactionCommitError( return; } - Write( + WriteEvent( SqlClientTransactionCommitError.Name, new SqlClientTransactionCommitError ( @@ -403,7 +404,7 @@ public void WriteTransactionRollbackAfter( return; } - Write( + WriteEvent( SqlClientTransactionRollbackAfter.Name, new SqlClientTransactionRollbackAfter ( @@ -433,7 +434,7 @@ public Guid WriteTransactionRollbackBefore( Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientTransactionRollbackBefore.Name, new SqlClientTransactionRollbackBefore ( @@ -465,7 +466,7 @@ public void WriteTransactionRollbackError( return; } - Write( + WriteEvent( SqlClientTransactionRollbackError.Name, new SqlClientTransactionRollbackError ( @@ -481,6 +482,17 @@ public void WriteTransactionRollbackError( ); } +#if NET + [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026", + Justification = "SqlClient does no reflection here; subscribers that reflect over payloads are responsible for their own reflection.")] + // The annotation on T is required by Write and keeps payload public properties for subscribers that reflect. + private void WriteEvent<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] T>(string name, T payload) => + Write(name, payload); +#else + private void WriteEvent(string name, T payload) => + Write(name, payload); +#endif + #if NET private void SqlDiagnosticListener_UnloadingAssemblyLoadContext(AssemblyLoadContext obj) => Dispose();