diff --git a/.config/LocalizationValidationAllowlist.json b/.config/LocalizationValidationAllowlist.json new file mode 100644 index 0000000000..0a3cbfea6b --- /dev/null +++ b/.config/LocalizationValidationAllowlist.json @@ -0,0 +1,110 @@ +{ + "_comment": [ + "These culture/key pairs intentionally match the English source text.", + "Each pair was verified in the internal LCL source as localized (Stat=Loc, Orig=New). Remove an entry when its localized value changes." + ], + "AllowedEnglishValueMatches": { + "Strings.cs.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_Data", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.de.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId" + ], + "Strings.es.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass" + ], + "Strings.fr.resx": [ + "DataCategory_InfoMessage", + "DataCategory_Notification", + "DataCategory_Source", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SqlMisc_NullString" + ], + "Strings.it.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.ja.resx": [ + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId" + ], + "Strings.ko.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.pl.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.pt-BR.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_Pooling", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass" + ], + "Strings.ru.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExOriginalClientConnectionId" + ], + "Strings.tr.resx": [ + "ADP_InvalidMultipartName", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SqlMisc_NullString" + ], + "Strings.zh-Hans.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ], + "Strings.zh-Hant.resx": [ + "DataCategory_InfoMessage", + "DataCategory_StatementCompleted", + "DataCategory_Xml", + "SQL_ExClientConnectionId", + "SQL_ExErrorNumberStateClass", + "SqlMisc_NullString" + ] + } +} diff --git a/.config/PolicheckExclusions.xml b/.config/PolicheckExclusions.xml index a4269513d1..e1129013ba 100644 --- a/.config/PolicheckExclusions.xml +++ b/.config/PolicheckExclusions.xml @@ -1,5 +1,5 @@ SRC/MICROSOFT.DATA.SQLCLIENT/TESTS .YML|.MD|.SQL - NOTICE.TXT|SQLDATAADAPTER.CS + NOTICE.TXT|SQLDATAADAPTER.CS|SQLDATAADAPTER.XML \ No newline at end of file diff --git a/.config/guardian/.gdnbaselines b/.config/guardian/.gdnbaselines index 10698219a1..e5c5ff89b2 100644 --- a/.config/guardian/.gdnbaselines +++ b/.config/guardian/.gdnbaselines @@ -8,7 +8,7 @@ "default": { "name": "default", "createdDate": "2026-07-23 11:29:23Z", - "lastUpdatedDate": "2026-07-23 11:29:23Z" + "lastUpdatedDate": "2026-08-28 14:33:24Z" } }, "results": { @@ -103,10 +103,15 @@ "0d5b851e97bdb0eb8931e6f326718a657f9cd46092eb8a2a2d414be2147a797c", "e6c0cd6ef2433a42c95a2939cce740019ecda3fcfde64a3a0f21661e6ff27f71" ], + "target": "src/Microsoft.Data.SqlClient/tests/ManualTests/makepfxcert.ps1", + "line": 145, + "uriBaseId": "file:///D:/a/_work/1/s/", "memberOf": [ "default" ], - "createdDate": "2026-07-23 11:29:23Z" + "tool": "psscriptanalyzer", + "ruleId": "PSAvoidUsingConvertToSecureStringWithPlainText", + "createdDate": "2026-08-28 12:58:19Z" }, "27cf35f7df3f630fab489573ec19318f563e042424ca30625e9fe08407d74bdf": { "signature": "27cf35f7df3f630fab489573ec19318f563e042424ca30625e9fe08407d74bdf", @@ -119,6 +124,664 @@ "default" ], "createdDate": "2026-07-23 11:29:23Z" + }, + "1e0989a7cdd65afb10dd3787a2ed33e9f737e6dd049524edbf76ba1daadf6ef8": { + "signature": "1e0989a7cdd65afb10dd3787a2ed33e9f737e6dd049524edbf76ba1daadf6ef8", + "alternativeSignatures": [ + "47067564034219f2cf40604fcd1beadb34ebe1b960cc75600796c8a0f4565604" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/src/Utils.cs", + "line": 72, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 14:17:20Z" + }, + "e5cd66384b36741191c98844947e6b857140c09b2c352b180664f8b4829c3e4c": { + "signature": "e5cd66384b36741191c98844947e6b857140c09b2c352b180664f8b4829c3e4c", + "alternativeSignatures": [ + "07fc741e29b6f1d01d84d3290b8c53dc7f22036a8313d1f41acdca253aa3e254" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Resources/StringsHelper.cs", + "line": 90, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "281a106076dd3d70aefcd230a90cef542f52488fb3ce164c94b213ededa12da5": { + "signature": "281a106076dd3d70aefcd230a90cef542f52488fb3ce164c94b213ededa12da5", + "alternativeSignatures": [ + "42cf7833cc5d63447162200f8926b57746ad3f6f2bd43318f0e606f022933c3e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/AzureAttestationBasedEnclaveProvider.cs", + "line": 215, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "fcda2db01b63d59f5a4aa73cb780405887f4616f26f9e23dfae52195cab4994e": { + "signature": "fcda2db01b63d59f5a4aa73cb780405887f4616f26f9e23dfae52195cab4994e", + "alternativeSignatures": [ + "9134dead4de902cc02f5f2e7785d84b422f31847f237ba6bcbaeb823e228fd7d" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveProviderBase.cs", + "line": 170, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "4ff2eb551fd17b4d3239b89cf97bfb09507cdfb631cb4787adc3a4f195e8bac7": { + "signature": "4ff2eb551fd17b4d3239b89cf97bfb09507cdfb631cb4787adc3a4f195e8bac7", + "alternativeSignatures": [ + "3b7985cbb5123ba5b91edc3e36a952d1f9d579943820f3c3e870095c7e264cc4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/NoneAttestationEnclaveProvider.cs", + "line": 43, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8f3e9755a800f590583d9e7ea2028c5d23ee2008450af7d55daea17d5fbecfa5": { + "signature": "8f3e9755a800f590583d9e7ea2028c5d23ee2008450af7d55daea17d5fbecfa5", + "alternativeSignatures": [ + "4531f356921a98edacfca7c32eb389b459c02014ea3db2d2d899ce65a87d52ca" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlAeadAes256CbcHmac256EncryptionKey.cs", + "line": 94, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "6ba87c464ec17dc52c0304f5cee224f8d3233507a79211916901a7b9d0d0908c": { + "signature": "6ba87c464ec17dc52c0304f5cee224f8d3233507a79211916901a7b9d0d0908c", + "alternativeSignatures": [ + "fcd1d28e2fe4772861caa303138474dc79869cc77079f79b55eda1612a4c4693" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlAuthenticationProviderManager.cs", + "line": 353, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "ffe242f34f321ccf4d2e727c9812baaf006e4ad122d314e02dd287f388b5b176": { + "signature": "ffe242f34f321ccf4d2e727c9812baaf006e4ad122d314e02dd287f388b5b176", + "alternativeSignatures": [ + "ca2f6e8136bafc65dc12eb6236123ee0877de7a1b48e810c36b7feae643b9654" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs", + "line": 1049, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "b73abd622849352bbe6c5188f106d4a2f9a1c4f650b7d2ba8f383653909a9479": { + "signature": "b73abd622849352bbe6c5188f106d4a2f9a1c4f650b7d2ba8f383653909a9479", + "alternativeSignatures": [ + "636e8fa98391919cfbd7f27792cac5c7806dbd7ec3ab0506b27fbbe52ee9cd8b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlClientPermission.netfx.cs", + "line": 144, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "cdbeec94358c58eb2960ea291f7f804e5d24e15807bc5b4a5ce259782727cd9a": { + "signature": "cdbeec94358c58eb2960ea291f7f804e5d24e15807bc5b4a5ce259782727cd9a", + "alternativeSignatures": [ + "8fa2f809347c794dbb3659a25cb2ea3a15df3a64a86f5a78f72b39c63f6b8319" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs", + "line": 2336, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "01c8d80a7268d44c7d8e478f887e804d8819cc2382ad1d845a415d97ead39d2e": { + "signature": "01c8d80a7268d44c7d8e478f887e804d8819cc2382ad1d845a415d97ead39d2e", + "alternativeSignatures": [ + "00aae68e847dfaed6240e67d9f0d1f5f64b9a0343c185c69f7dae148ebf2dc35" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionEncryptOption.cs", + "line": 57, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "5ef60ae6fdf13d9fcebe7631af27f7ea23d3f198215a19c8752bf5f8cdee8dc9": { + "signature": "5ef60ae6fdf13d9fcebe7631af27f7ea23d3f198215a19c8752bf5f8cdee8dc9", + "alternativeSignatures": [ + "17d62daf6be556a78b7a342be79f5038d7f1831851722a69398b996cecd57bd8" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionEncryptOption.cs", + "line": 108, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f99754da199aaa7a5e510552f0b4e851029c149dced1d5196eba374b03f0450d": { + "signature": "f99754da199aaa7a5e510552f0b4e851029c149dced1d5196eba374b03f0450d", + "alternativeSignatures": [ + "2852f83af1a5eacc738153cae383e7e216179f3573e9082879dddb48e32a260c" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs", + "line": 1638, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "3dc0ef1e00dd1aed1bd9b6a2e9c06c4f8b24bf368419a2928feaab51252a3b47": { + "signature": "3dc0ef1e00dd1aed1bd9b6a2e9c06c4f8b24bf368419a2928feaab51252a3b47", + "alternativeSignatures": [ + "6a46ad5f8328cb647c28327bb4260335dab6e381ee816f60e371f3f3c4f348b0" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.cs", + "line": 1640, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d0e489f06819d4a2e78e99be8aedb5d683f8c787cfc07d5c25689bc4e4e3f5f6": { + "signature": "d0e489f06819d4a2e78e99be8aedb5d683f8c787cfc07d5c25689bc4e4e3f5f6", + "alternativeSignatures": [ + "524203b79cc017dc30443712f932710574a9a39d5f7251a4c9354f3cdd21b36b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.Debug.cs", + "line": 59, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "6d1f24e834de7bf17aa69abdd0fe79751a4afcc083253e84b577c813a19f46fb": { + "signature": "6d1f24e834de7bf17aa69abdd0fe79751a4afcc083253e84b577c813a19f46fb", + "alternativeSignatures": [ + "9b805fc1d43486b21ec75d68dd2e4b19d5e7af7c954c546389307bd689a2930b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionOptions.Debug.cs", + "line": 59, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d245c0ad54d0229b6ea7a194bd42c40011b734ee5ef92dfce42c9a40096c906d": { + "signature": "d245c0ad54d0229b6ea7a194bd42c40011b734ee5ef92dfce42c9a40096c906d", + "alternativeSignatures": [ + "472b266989720cacdc2666a97234830e954f84346ba13ee028d9f0dbac3d5d82" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDataReader.cs", + "line": 2801, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "68fc925a125fdb6381d42edaaae658cb5c23c5bbef262cbb569215d839e3a9b3": { + "signature": "68fc925a125fdb6381d42edaaae658cb5c23c5bbef262cbb569215d839e3a9b3", + "alternativeSignatures": [ + "1a6475a1a8210fd0e0d4807a4c5d4e1017da257dbfa97b17c8e3382c1684a34a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDependency.cs", + "line": 644, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA2219", + "createdDate": "2026-08-28 13:55:54Z" + }, + "df9958d713606f4b2d800a8f5b33e4839cde7d9e7514f732e0b999f4e7df0cb0": { + "signature": "df9958d713606f4b2d800a8f5b33e4839cde7d9e7514f732e0b999f4e7df0cb0", + "alternativeSignatures": [ + "b44a5e32ae614b1bdda97a1a634dad5a4ab4ede29b463dd3196f309db10e1d15" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlDependency.cs", + "line": 1222, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "e318e84cf65f756c454457b437010a21fac2f27f2fa7378c3befc809b97369be": { + "signature": "e318e84cf65f756c454457b437010a21fac2f27f2fa7378c3befc809b97369be", + "alternativeSignatures": [ + "d8118425a1409e87f5983a840ac22cc663a7ab494bb914b312f7b32adb84d5f5" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlEnums.cs", + "line": 1134, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "467568c2128c495889d899ef783ccca652f3b355aa8fe815d4da1b55b6deab95": { + "signature": "467568c2128c495889d899ef783ccca652f3b355aa8fe815d4da1b55b6deab95", + "alternativeSignatures": [ + "8962958e2e5e468cc56038a04d02476e352bf6f1d48104f51c13990d364b0c64" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlException.cs", + "line": 164, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "727c936839845e9a68ff00fbc69abb5a132769506d3cfe87692612a2addac855": { + "signature": "727c936839845e9a68ff00fbc69abb5a132769506d3cfe87692612a2addac855", + "alternativeSignatures": [ + "2450ed8367a7fac65b41524a145bf097f2dee5794752124fefa3a635057946bd" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs", + "line": 114, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8c6b44ec1f44fbe00469a48beb3f9eee61c8e31e237c53ba372a1abfc5ea481f": { + "signature": "8c6b44ec1f44fbe00469a48beb3f9eee61c8e31e237c53ba372a1abfc5ea481f", + "alternativeSignatures": [ + "7a3c05145c302720ca6feadabcacbb11303442b2289fb06796e21b713fd63717" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs", + "line": 572, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8a5592e024f45a9bdce3315e108cccfafe99b184dbcb4e50d59d783fa3db3942": { + "signature": "8a5592e024f45a9bdce3315e108cccfafe99b184dbcb4e50d59d783fa3db3942", + "alternativeSignatures": [ + "929389afe5818dcc5113299f0e0263eca26ef989785742ac7e46b088d5cae598" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlParameter.cs", + "line": 2364, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "770e1d8ef5a06c9b4b552dc9c8bf000f572d633bf11ded7e749b3ac9c07d208b": { + "signature": "770e1d8ef5a06c9b4b552dc9c8bf000f572d633bf11ded7e749b3ac9c07d208b", + "alternativeSignatures": [ + "4cd0e3d7087eaa0eaf05bec58d32fd71b660033f7bf3984392998d0b9946fb47" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlSecurityUtility.cs", + "line": 389, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8502b61241cdbfac98f7c84b8c16d4d4ec0ea2185018d9fce710e2bac445e8b0": { + "signature": "8502b61241cdbfac98f7c84b8c16d4d4ec0ea2185018d9fce710e2bac445e8b0", + "alternativeSignatures": [ + "afed061c02d7b602c516f09952197b58c6c1cdfa0e07ee3625af48287869b2c5" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs", + "line": 1749, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "a83b0d426733a42a34b4569643e60513b598035d6807fa8cda6f4a3501084929": { + "signature": "a83b0d426733a42a34b4569643e60513b598035d6807fa8cda6f4a3501084929", + "alternativeSignatures": [ + "3614422f0f09ab4fff1a0195a4f40acd53bf55e4781153ced9d736a801fc6aa4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlUtil.cs", + "line": 1875, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d36be8dd5189d09c4b5db9176bda7628839f0ba608f31ba449f1203ca9c30c09": { + "signature": "d36be8dd5189d09c4b5db9176bda7628839f0ba608f31ba449f1203ca9c30c09", + "alternativeSignatures": [ + "fcf7dc6efdfecd535087f2361d43cfa599cbe83b52717beb36c052dadbbc5a5d" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs", + "line": 2309, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "b19ca78715cc12a2051b38a495f41c060259e46d893be38e6dba447e8a87cc02": { + "signature": "b19ca78715cc12a2051b38a495f41c060259e46d893be38e6dba447e8a87cc02", + "alternativeSignatures": [ + "efc4d3f86b80b0d8392e7fa74b15ca51411078ed8f249b0f30911036530bc0b4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs", + "line": 3918, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1304", + "createdDate": "2026-08-28 13:55:54Z" + }, + "119b9514aeb84d5e34b5d7b1520378d4a2d017902d7d1bd572404f1b3186c59c": { + "signature": "119b9514aeb84d5e34b5d7b1520378d4a2d017902d7d1bd572404f1b3186c59c", + "alternativeSignatures": [ + "57389b1211c68b1c8ed21fb3439d1dd9bd8a8e25677ed63b1db38ff7bd3b0c6b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs", + "line": 4403, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "533d9dc25ef8183859aeed955b720d435e3035702b3b7c10d12204170815ec04": { + "signature": "533d9dc25ef8183859aeed955b720d435e3035702b3b7c10d12204170815ec04", + "alternativeSignatures": [ + "55c532f39c15ec069c540c661f1ccb7371ee8c03f74fad4515d9cd94db8bcca1" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserStateObjectNative.cs", + "line": 100, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "90dd8eeec55b68eaf94f971d1af06723d22efc0f7cb5fd15870765e6e6602ff9": { + "signature": "90dd8eeec55b68eaf94f971d1af06723d22efc0f7cb5fd15870765e6e6602ff9", + "alternativeSignatures": [ + "ac02e05713e85c6086fc30c51a7083a159329f0a222c9d0be513e6da9fda0300" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/VirtualSecureModeEnclaveProvider.cs", + "line": 84, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f1c711a9df14d19ac6682356d6648111a1e3c2ae051812e8f04a7466df045aa4": { + "signature": "f1c711a9df14d19ac6682356d6648111a1e3c2ae051812e8f04a7466df045aa4", + "alternativeSignatures": [ + "1e752a5b4246f05f58da2adde778eb3eb46238ad51a6a2013d3fd9d80aaf5422" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/VirtualSecureModeEnclaveProviderBase.cs", + "line": 488, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "d159ab04bf10a650f1f43064602246acfa671e4f1f7ba07faf5620ab13043926": { + "signature": "d159ab04bf10a650f1f43064602246acfa671e4f1f7ba07faf5620ab13043926", + "alternativeSignatures": [ + "74e1a9e9972040959e4176eb8dc981d8394545eaf86f30e059f9814f545bea67" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/ConnectionString/DbConnectionString.netfx.cs", + "line": 374, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "761ae24086cbfbeb9667e20ab45b174613bd156d0dc7bde56a37571db11cd779": { + "signature": "761ae24086cbfbeb9667e20ab45b174613bd156d0dc7bde56a37571db11cd779", + "alternativeSignatures": [ + "1793d6a0e1d5ee495ced5ce18af4bb1c6f1635ce89d6dfe9d840b84673ccbe7a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs", + "line": 4130, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "92759af0d8bc9a56339a09bfb5ae1e0adbe2c0aa7697911d0f14941ba65a1e67": { + "signature": "92759af0d8bc9a56339a09bfb5ae1e0adbe2c0aa7697911d0f14941ba65a1e67", + "alternativeSignatures": [ + "f20cbad7a6ed702f47f5d26f6e56844ce9f1e5a7d03152361b1818d4aef03058" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/DbConnectionPoolAuthenticationContextKey.cs", + "line": 83, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "8409f840dce0e042b55250ce12045afaabc1e2e48f443e43ec0e493517e3e38d": { + "signature": "8409f840dce0e042b55250ce12045afaabc1e2e48f443e43ec0e493517e3e38d", + "alternativeSignatures": [ + "b607bb6c9aded6c9d9d1da95e9f49b9a17ec42880fe59156cfca1d2f088fec35" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniCommon.netcore.cs", + "line": 148, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "1152991aab5d089b6b086e389b075de228b319c82a577dae0afe850956202b12": { + "signature": "1152991aab5d089b6b086e389b075de228b319c82a577dae0afe850956202b12", + "alternativeSignatures": [ + "95a2b83edbb49524b5834cba58e1c1670ad5397cca2bc8ed4912f02c7b885acd" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs", + "line": 150, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "66052c0cd9b0dade1d5d86703650fa9c8d46f7977b60148aea67f371393e32b3": { + "signature": "66052c0cd9b0dade1d5d86703650fa9c8d46f7977b60148aea67f371393e32b3", + "alternativeSignatures": [ + "d67c06a92599b72202abeed718e385e4621947195fc3affc730a1066b55f5cb6" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs", + "line": 731, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:55:54Z" + }, + "f236b2c674fc8574908a9b254e10d9a78f4df86278d9cfe52f5b4d072689829b": { + "signature": "f236b2c674fc8574908a9b254e10d9a78f4df86278d9cfe52f5b4d072689829b", + "alternativeSignatures": [ + "fabdb2e276df6d043b029af9a40c831c8f9f7131dec81706dd664cd5d913f32e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniTcpHandle.netcore.cs", + "line": 658, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "424ec1510b6c1352e0731f19bd7a6c03f1da3bab869fa520046b0080cb7b5d70": { + "signature": "424ec1510b6c1352e0731f19bd7a6c03f1da3bab869fa520046b0080cb7b5d70", + "alternativeSignatures": [ + "4dac74c5b9483f0a4b1f7e76f91002aa485fd1b83b19d5823169b0c15eaace8b" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SsrpClient.netcore.cs", + "line": 80, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "76482116b8cae39bdad9b92782ab216ff958f6578405f93879733326a4283bfe": { + "signature": "76482116b8cae39bdad9b92782ab216ff958f6578405f93879733326a4283bfe", + "alternativeSignatures": [ + "a815e32ff0ad176d42719f84cb015df61c89e9539fecbf58da169993e789b682" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs", + "line": 308, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:55:54Z" + }, + "dd455bea13e4bd70696d1b0f4c890ca1ea42e18643bbe6b572989d84ac188f06": { + "signature": "dd455bea13e4bd70696d1b0f4c890ca1ea42e18643bbe6b572989d84ac188f06", + "alternativeSignatures": [ + "0444dab1ebebce0b0e0bfb453d8513e9e8108aad22055bd038c873a8a8bc1f0e" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs", + "line": 629, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1309", + "createdDate": "2026-08-28 13:36:06Z" + }, + "09526af76b9a6ad21a4841ea50bb51e72925789c5f0f733650ac946cc44060a9": { + "signature": "09526af76b9a6ad21a4841ea50bb51e72925789c5f0f733650ac946cc44060a9", + "alternativeSignatures": [ + "2a3fb00242d533ee3253b91c42ba7882b9c35ada1635e3470c481c6b43baa0e6" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/SqlUserDefinedAggregateAttribute.netstandard.cs", + "line": 61, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "6fec84f5f9e3087c484965319896c91b033557b8bcef7761b0ae74e1baf0ad7d": { + "signature": "6fec84f5f9e3087c484965319896c91b033557b8bcef7761b0ae74e1baf0ad7d", + "alternativeSignatures": [ + "efa9c07656c41d1d2367a89a7146fda28f3b584e2654693b4954d40bfafecb8f" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/SqlUserDefinedTypeAttribute.netstandard.cs", + "line": 73, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "1de5740b9e122c2d8bda4bfe66c94764a6192376f753a7234bac91e1fc28e5f6": { + "signature": "1de5740b9e122c2d8bda4bfe66c94764a6192376f753a7234bac91e1fc28e5f6", + "alternativeSignatures": [ + "14c7177139597c2ab94cf632b76bc03c4f2c252691b30a7ba69957e047b400c4" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.SqlServer.Server/StringsHelper.netstandard.cs", + "line": 130, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:10:07Z" + }, + "0ca59be802da38e82370950b24300deae99834aa9a3cee38e064d1f17c5942a9": { + "signature": "0ca59be802da38e82370950b24300deae99834aa9a3cee38e064d1f17c5942a9", + "alternativeSignatures": [ + "26425ecc80bd3289865ce79ae6471c4ab21cde25acc1cf2067717fbf6d600a6a" + ], + "target": "file:///C:/__w/1/s/src/Microsoft.Data.SqlClient.Internal/Logging/src/SqlClientEventSource.cs", + "line": 1995, + "memberOf": [ + "default" + ], + "tool": "roslynanalyzers", + "ruleId": "CA1305", + "createdDate": "2026-08-28 13:09:32Z" } } -} \ No newline at end of file +} diff --git a/.gitattributes b/.gitattributes index a1d79dfeee..ab4f136a23 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,4 +2,4 @@ * text=auto # Treat workflow lock files as generated -.github/workflows/*.lock.yml linguist-generated=true merge=ours +.github/workflows/*.lock.yml linguist-generated=true diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 64a0a0a93a..cdcb39f6a2 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -5,10 +5,10 @@ "version": "v9.0.0", "sha": "3a2844b7e9c422d3c10d287c895573f7108da1b3" }, - "github/gh-aw-actions/setup@v0.80.9": { + "github/gh-aw-actions/setup@v0.88.2": { "repo": "github/gh-aw-actions/setup", - "version": "v0.80.9", - "sha": "8c7d04ebf1ece56cd381446125da3e0f6896294a" + "version": "v0.88.2", + "sha": "9271a1804551c0dc4fb0085a97979950aa2f8489" } } } diff --git a/.github/instructions/ado-pipelines.instructions.md b/.github/instructions/ado-pipelines.instructions.md index 3e5a66af78..4fd07cfc88 100644 --- a/.github/instructions/ado-pipelines.instructions.md +++ b/.github/instructions/ado-pipelines.instructions.md @@ -93,7 +93,7 @@ Test filters — default excludes `failing` and `flaky` categories: - `nonwindowstests` / `nonlinuxtests` — OS-specific exclusions Flaky test quarantine: -- Quarantined tests (`[Trait("Category", "flaky")]`) run in separate steps after main tests +- Quarantined tests (`[Trait("category", "flaky")]`) run in separate steps after main tests - Main test runs are not blocked by flaky failures - No code coverage collected for flaky runs - Configured in `common/templates/steps/build-and-run-tests-netcore-step.yml`, `build-and-run-tests-netfx-step.yml`, and `run-all-tests-step.yml` diff --git a/.github/instructions/connection-pooling.instructions.md b/.github/instructions/connection-pooling.instructions.md index 6464fdedd0..95019b996d 100644 --- a/.github/instructions/connection-pooling.instructions.md +++ b/.github/instructions/connection-pooling.instructions.md @@ -124,13 +124,6 @@ private readonly ChannelWriter _idleConnectionWriter; await _idleConnectionReader.WaitToReadAsync(token); ``` -### Sync Over Async Protection -```csharp -// Prevent thread pool starvation -private static SemaphoreSlim _syncOverAsyncSemaphore = - new(Math.Max(1, Environment.ProcessorCount / 2)); -``` - ## Best Practices ### Application Design diff --git a/.github/instructions/onebranch-pipeline-design.instructions.md b/.github/instructions/onebranch-pipeline-design.instructions.md index 0b647b6be0..7d42d279e7 100644 --- a/.github/instructions/onebranch-pipeline-design.instructions.md +++ b/.github/instructions/onebranch-pipeline-design.instructions.md @@ -26,15 +26,19 @@ Respect this graph when modifying build stages: 5. `Microsoft.Data.SqlClient.Extensions.Azure` — depends on Abstractions + Logging 6. `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` — depends on SqlClient + Abstractions + Logging +## Localization Validation + +The SqlClient build job runs `steps/validate-localization-step.yml` before building the driver. Validation always fails the build for missing or obsolete keys, empty localized values whose English value is non-empty, and untranslated resources. Approved identical translations are listed by culture and resource key in `.config/LocalizationValidationAllowlist.json`. + ## Build Stages -Defined in `stages/build-stages.yml`. Four build stages plus validation, ordered by dependency: +Defined in `stages/build-stages.yml`. Four build stages plus package validation are ordered by dependency: - **`build_independent`** (Stage 1) — Logging and SqlServer.Server in parallel; no inter-package dependencies - **`build_abstractions`** (Stage 2) — Abstractions; `dependsOn: build_independent`; downloads Logging artifact - **`build_dependent`** (Stage 3) — SqlClient and Extensions.Azure in parallel; `dependsOn: build_abstractions`; downloads Abstractions + Logging artifacts - **`build_addons`** (Stage 4) — AKV Provider; `dependsOn: build_dependent`; downloads SqlClient + Abstractions + Logging artifacts -- **`sqlclient_package_validation`** — Validates signed SqlClient package; `dependsOn: build_dependent`; runs in parallel with Stage 4 +- **`package_validation`** (Stage 5) — Validates every package produced by the run; `dependsOn` all four build stages plus `compute_versions` Each build job copies PDB files into `$(JOB_OUTPUT)/symbols/` so they are included in the auto-published pipeline artifact alongside the NuGet packages in `$(JOB_OUTPUT)/packages/`. @@ -46,7 +50,7 @@ Stage conditional rules: ## Job Templates - **`build-buildproj-job.yml`** — Shared build.proj-driven package job used for all shipped packages. Flow: build via `build.proj` → optional ESRP DLL signing → pack via `build.proj` → optional ESRP NuGet signing → copy outputs for APIScan/artifacts -- **`validate-signed-package-job.yml`** — Validates signed MDS package (signature, strong names, folder structure, target frameworks) +- **`validate-packages-job.yml`** — Validates every package produced by the run. Downloads all package artifacts into one tree and validates them together, so `tools/PackageValidator` can apply its cross-package rules (the SqlClient family must share one version, and inter-package dependency ranges must agree); validating per package would silently skip those findings. Runs on Windows because Authenticode verification has no Linux equivalent - **`publish-nuget-package-job.yml`** — Reusable release job using OneBranch `templateContext.type: releaseJob` with `inputs` for artifact download; pushes via `NuGetCommand@2` - **`publish-symbols-job.yml`** — Reusable symbols job: downloads a build artifact, locates PDBs under `symbols/`, and invokes `publish-symbols-step.yml` @@ -56,6 +60,19 @@ When adding a new package to the OneBranch flow: - Add version variables to `variables/common-variables.yml` - Add artifact name variables to `variables/onebranch-variables.yml` +## Package Validation Stage + +- Defined in `stages/build-stages.yml`; produces stage `package_validation` +- Consumes the package and file versions published by `compute_versions` and asserts the produced packages carry exactly those values, so nothing is re-derived +- All packages are validated together in one job so `tools/PackageValidator` can apply cross-package rules; the SqlServer artifact and its expectations are conditional on `buildSqlServer` +- Expectations use the validator's `[id=]value` form: the SqlClient family version is applied as a wildcard (proving the family agrees, and catching the case where all packages are consistently wrong), with `Microsoft.SqlServer.Server` as a per-id override +- When SqlServer is not built its expectations are **omitted entirely** rather than passed empty — the validator rejects an expectation with an empty value +- Gate categories are derived from `isOfficial`: `error`, `missing-symbols`, `dependency-inconsistency`, `delay-signed`, and `unsigned` always, plus `package-unsigned` on official runs only. The `error` severity covers only error-severity findings, so each warning/info category must be named explicitly — `missing-symbols`, `dependency-inconsistency`, and `delay-signed` are warnings, and `unsigned` and `package-unsigned` are info. Strong-name signing is unconditional in `build-buildproj-step.yml`, so the two strong-name categories gate everywhere; NuGet package signing is ESRP and official-only, so `package-unsigned` would fire on every non-official run +- The validator runs twice: once with `--json` and no gate so the report exists even for a failing run, then once human-readable with the gate so failures appear in the job log +- Signature verification (`dotnet nuget verify --all`, Authenticode) runs on official builds only, and verifies that signatures are *trusted* — PackageValidator reports only their presence, from metadata +- The release stage `dependsOn: package_validation`, so a package that fails validation is never published +- Step and job logic lives in `scripts/validate-packages.ps1`, `scripts/verify-package-signatures.ps1`, and `scripts/verify-assembly-signatures.ps1`, each with Pester tests under `scripts/tests/` + ## Symbols Publishing Stage - Defined in `stages/publish-symbols-stage.yml`; produces stage `publish_symbols` @@ -108,7 +125,8 @@ When `isPreview` is true, pipeline resolves `effective*Version` variables to pre - Variable chain: pipeline YAML → `variables/onebranch-variables.yml` → `variables/common-variables.yml` - All package versions (GA, preview, assembly file) centralized in `variables/common-variables.yml` -- `effective*Version` pipeline variables map to selected version set based on `isPreview` +- The `compute_versions` stage reads canonical versions from MSBuild and publishes effective package, + file-build, and APIScan registration versions for downstream stages - Artifact name variables defined in `variables/onebranch-variables.yml` following `drop__` pattern - `assemblyBuildNumber` derived from first segment of `Build.BuildNumber` only (16-bit limit) - When adding a new package, add GA version, preview version, and assembly file version entries @@ -128,9 +146,13 @@ Variable groups: ## SDL and Compliance - TSA: enabled only in official pipeline; disabled in non-official to avoid spurious alerts -- ApiScan: enabled in both; currently `break: false` pending package registration -- Each build job sets `ob_sdl_apiscan_softwareFolder` to `$(JOB_OUTPUT)/assemblies` and `ob_sdl_apiscan_symbolsFolder` to `$(JOB_OUTPUT)/symbols` +- ApiScan: enabled in both; `break` follows the `breakOnSdlError` parameter +- Each package is registered with APIScan under its own name/version pair, so the `globalSdl.apiscan` blocks deliberately omit `softwareName`/`versionNumber`. `build-buildproj-job.yml` is the single place they are set, via `ob_sdl_apiscan_softwareName` (the package's `packageFullName`) and `ob_sdl_apiscan_versionNumber` (the `apiScanSoftwareVersion` parameter) +- `compute-versions.ps1` derives APIScan registration versions as major.minor from the effective canonical package versions and publishes them as stage outputs. A package name/version pair must still be registered with APIScan before releasing a new major.minor. Consume these as runtime `$(...)` references so values such as `1.0` remain strings rather than being coerced to numbers by template expressions +- Jobs that produce no assemblies (symbol publishing, signed-package validation, version computation) set `ob_sdl_apiscan_enabled: false` rather than reporting a name/version +- Each build job also sets `ob_sdl_apiscan_softwareFolder` and `ob_sdl_apiscan_symbolsFolder` to its per-package `apiScan//dlls` and `apiScan//pdbs` paths - CodeQL, SBOM, Policheck (`break: true`): enabled in both pipelines +- SBOM package name/version are resolvable **only** from the pipeline's `globalSdl.sbom` block — OneBranch's artifact-publishing path reads `globalSdl.sbom.packageName`/`packageVersion` directly and has no per-job equivalent (the `templateContext.sdl.sbom` override only applies to the native 1ES Stages entry point, which this repo does not use). Because the pipeline produces six differently-named and independently-versioned packages, `globalSdl.sbom` indirects through the `$(sbomPackageName)` / `$(sbomPackageVersion)` variables, which each build job sets to its own `packageFullName` and computed `packageVersion`. Jobs that publish no packages (version computation, symbol publishing) set `ob_sdl_sbom_enabled: false` alongside their existing APIScan/BinSkim opt-outs, so the variables never need pipeline-level defaults - asyncSdl `enabled: false` in both; individual sub-tools (CredScan, BinSkim, Armory, Roslyn) configured underneath - Policheck exclusions: `$(REPO_ROOT)\.config\PolicheckExclusions.xml` - CredScan suppressions: `$(REPO_ROOT)/.config/CredScanSuppressions.json` diff --git a/.github/instructions/sqlclient-package-versions.instructions.md b/.github/instructions/sqlclient-package-versions.instructions.md index 1692082a68..691367d9de 100644 --- a/.github/instructions/sqlclient-package-versions.instructions.md +++ b/.github/instructions/sqlclient-package-versions.instructions.md @@ -40,7 +40,7 @@ Each `Versions.props` uses a 3-tier `` block: | Priority | Condition | PackageVersion | FileVersion | |----------|-----------|----------------|-------------| | 1 | `PackageVersion` explicitly provided | Used as-is | Strip prerelease + append BuildNumber | -| 2 | `BuildNumber` provided (non-zero) | `NextVersion[-BuildSuffix+BuildNumber]` | `NextVersion.Split('-')[0].BuildNumber` | +| 2 | `BuildNumber` provided (non-zero) | `NextVersion[-BuildSuffix]`, then `.BuildNumber` appended if that carries a prerelease tag | `NextVersion.Split('-')[0].BuildNumber` | | 3 | Nothing provided | `NextVersion-dev` | `NextVersion.Split('-')[0].0` | For every family package, `` is `SqlClient` (e.g. `-p:SqlClientPackageVersion=...`); for @@ -79,7 +79,7 @@ Microsoft.SqlServer.Server it is `SqlServer`. - Versions computed in `compute-versions-ci-stage.yml` (runs `GetVersions*` targets with `-p:BuildSuffix=pr -p:BuildNumber=...`) - Falls into Priority 2 with BuildSuffix present. -- **Result:** `7.1.0-preview1-pr15401` / FileVersion `7.1.0.15401` +- **Result:** `7.1.0-preview1-pr.15401` / FileVersion `7.1.0.15401` - Dependencies are project references — all packages built together in-tree. **Mode:** Package (PR package-ref validation) @@ -93,7 +93,7 @@ Microsoft.SqlServer.Server it is `SqlServer`. Same structure as PR but passes `buildSuffix: 'ci'` explicitly. -- **Result:** `7.1.0-preview1-ci15401` / FileVersion `7.1.0.15401` +- **Result:** `7.1.0-preview1-ci.15401` / FileVersion `7.1.0.15401` ### OneBranch Pipeline (official) @@ -104,8 +104,8 @@ Uses the full `compute-versions-stage.yml` machinery: #### Step A: Compute Versions (dedicated early stage) 1. Runs the `GetVersionsSqlClient` and `GetVersionsSqlServer` MSBuild targets against `build.proj`. -2. Each target calls `dotnet build -getProperty:PackageVersion` with `BuildNumber` but **no BuildSuffix**. -3. Falls into Priority 2 without BuildSuffix → `PackageVersion = NextVersion` as-is (e.g. `7.1.0-preview1`). +2. Each target calls `dotnet build -getProperty:PackageVersion` and `-getProperty:FileVersion` with `BuildNumber` but **no BuildSuffix**. +3. Falls into Priority 2 without BuildSuffix. `NextVersion` already carries a prerelease tag on `main`, so the build number is appended (e.g. `7.1.0-preview1.26238.3`); on a release branch the stable `NextVersion` is used as-is (e.g. `7.1.0`). 4. `GetVersionsSqlServer` also extracts `SqlServerPublishedVersion` (the SqlClient family has no published version). #### Step B: Resolve Effective Versions @@ -136,15 +136,12 @@ Each downstream build job receives: Since an explicit `PackageVersion` is provided, Versions.props hits Priority 1 — uses the value verbatim. -#### Package Version Shapes: `addRevision` +#### Package Version Shapes -Both OneBranch pipelines expose `addRevision` (default `false`), which selects between two mutually -exclusive package version shapes. Their human-readable run name is `$(Year:YY)$(DayOfYear)$(Rev:.r)`, -and the compute stage receives both that run name (as `Build.BuildNumber`) and the globally unique -`Build.BuildId` (as the revision). - -**Default path — `addRevision: false`.** The pipeline run name is appended after any prerelease -suffix, reproducing the shape shipped by earlier previews. `Build.BuildId` is not used: +Both OneBranch pipelines use the human-readable run name `$(Year:YY)$(DayOfYear)$(Rev:.r)`, which the +compute stage receives as `Build.BuildNumber`. That run name drives the single supported package +version shape: it is appended after any prerelease suffix, reproducing the shape shipped by earlier +previews. - `1.2.3` stays `1.2.3` — non-preview releases are never stamped with a build number - `1.2.3-preview1` becomes `1.2.3-preview1.`, e.g. `7.1.0-preview3.26238.3` @@ -152,34 +149,15 @@ suffix, reproducing the shape shipped by earlier previews. `Build.BuildId` is no Note the asymmetry: the *package* version omits the build number for non-preview releases, but the *file* version always carries one in its fourth component. This keeps every shipped assembly -date-encoded and traceable to the run that produced it, while preserving the released package -version customers expect. - -**Opt-in path — `addRevision: true`.** Version revisions come from `Build.BuildId` instead, which is -mapped into the unsigned 16-bit file-version range before canonical file versions are evaluated: - -```text -revision = ((Build.BuildId - 1) % 65535) + 1 -``` - -Build IDs `1` through `65535` map directly; subsequent IDs wrap back through that range. The compute -stage logs the mapping whenever wrapping occurs because the revision can then collide with an earlier -run. The mapped value is inserted before any package prerelease suffix so package and file versions -use the same revision: - -- `1.2.3` becomes `1.2.3.` -- `1.2.3-preview1` becomes `1.2.3.-preview1` -- The matching file version is `1.2.3.` - -This shape exists for repeated test publishes of the same base version, where each run needs a -distinct package version. `Build.BuildNumber` is not used on this path. +date-encoded, while preserving the released package version customers expect. -Only packages built in the current run are revised or stamped. When `buildSqlServer` is `false`, the -effective SqlServer version remains `SqlServerPublishedVersion` so dependency restore continues to -request the package that actually exists on NuGet. +The file version's fourth component is only the *date* segment of the run name, because a four-part +file version cannot hold the full `.` value. Repeated runs on the same day therefore share +a file version even though their package versions differ. -An explicit four-part package version is also treated as the complete file version base by both -canonical Versions.props files; they do not append `FileVersionBuildNumber` as a fifth component. +Only packages built in the current run are stamped. When `buildSqlServer` is `false`, the effective +SqlServer version remains `SqlServerPublishedVersion` so dependency restore continues to request the +package that actually exists on NuGet. #### Summary diff --git a/.github/instructions/testing.instructions.md b/.github/instructions/testing.instructions.md index df6b909f8a..9f4032e070 100644 --- a/.github/instructions/testing.instructions.md +++ b/.github/instructions/testing.instructions.md @@ -69,7 +69,7 @@ Copy `config.default.jsonc` to `config.jsonc` and configure: ## Test Categories and Attributes ### Category Exclusions -Use `[Trait("Category", "...")]` (xUnit, used in both ManualTests and UnitTests) to mark test categories and exclusions: +Use `[Trait("category", "...")]` (xUnit, used in both ManualTests and UnitTests) to mark test categories and exclusions: | Category | Excluded On | Description | |----------|-------------|-------------| @@ -82,7 +82,7 @@ Use `[Trait("Category", "...")]` (xUnit, used in both ManualTests and UnitTests) | `flaky` | All platforms (quarantine) | Intermittently failing tests (see Quarantine Zone below) | ### Flaky Test Quarantine Zone -Tests that intermittently fail are quarantined with `[Trait("Category", "flaky")]`. Quarantined tests: +Tests that intermittently fail are quarantined with `[Trait("category", "flaky")]`. Quarantined tests: - Are **excluded** from regular test runs by the default filter: `category!=failing&category!=flaky` - Run in **separate quarantine pipeline steps** to track their status without blocking CI - Do **not** collect code coverage @@ -96,16 +96,16 @@ Tests that intermittently fail are quarantined with `[Trait("Category", "flaky") **How to quarantine:** ```csharp // For unit tests (xUnit Trait) -[Trait("Category", "flaky")] +[Trait("category", "flaky")] public class FlakyConnectionTests { ... } // For individual test methods -[Trait("Category", "flaky")] +[Trait("category", "flaky")] [ConditionalFact(...)] public async Task OpenAsync_TimingDependent_MayFail() { ... } ``` -**How to un-quarantine:** Remove the `[Trait("Category", "flaky")]` attribute once the root cause is fixed and the test passes consistently. +**How to un-quarantine:** Remove the `[Trait("category", "flaky")]` attribute once the root cause is fixed and the test passes consistently. ### Test Timeout Enforcement All test runs use `--blame-hang-timeout 10m` to kill tests that hang for more than 10 minutes. This is configured in `build.proj` and applied to all test targets. If a test is expected to run longer than 10 minutes, it must be restructured or split. @@ -120,11 +120,11 @@ This can be overridden via build property: `dotnet build build.proj -t:TestSqlCl ### Test Attributes ```csharp // Platform-specific exclusion -[Trait("Category", "nonlinuxtests")] +[Trait("category", "nonlinuxtests")] public void TestWindowsSpecificFeature() { ... } // Skip on .NET Framework -[Trait("Category", "nonnetfxtests")] +[Trait("category", "nonnetfxtests")] public void TestNetCoreOnlyFeature() { ... } // Conditional skip based on test configuration @@ -132,7 +132,7 @@ public void TestNetCoreOnlyFeature() { ... } public void TestRequiresDatabase() { ... } // Quarantined flaky test -[Trait("Category", "flaky")] +[Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] public void TestIntermittentlyFails() { ... } ``` diff --git a/.github/prompts/triage-pipeline-failures.prompt.md b/.github/prompts/triage-pipeline-failures.prompt.md index c4ff676d17..98c223c3d5 100644 --- a/.github/prompts/triage-pipeline-failures.prompt.md +++ b/.github/prompts/triage-pipeline-failures.prompt.md @@ -150,7 +150,7 @@ immediately-preceding in-scope build (the parent commit). A failure present befo ## Step 7 — Check quarantine status before acting -A test is already quarantined if it carries `[Trait("Category", "flaky")]`; those run +A test is already quarantined if it carries `[Trait("category", "flaky")]`; those run in a separate, non-blocking quarantine step (`TestFilters="category=flaky"`) while the regular step excludes `category!=failing&category!=flaky&category!=interactive`. @@ -175,7 +175,7 @@ the user which items to act on. For each item the user approves: 1. Prefer a **deterministic fix** that removes the race/isolation/timing dependency. -2. Otherwise **quarantine**: add `[Trait("Category", "flaky")]` plus a comment holding +2. Otherwise **quarantine**: add `[Trait("category", "flaky")]` plus a comment holding the observed failure signature (test name, assertion, key stack frames) and the root-cause reasoning. Mirror the style of existing quarantine comments in the test suite. 3. Cover both sync and async variants when the API has both. diff --git a/.github/scripts/check-milestone-branch.sh b/.github/scripts/check-milestone-branch.sh new file mode 100755 index 0000000000..3be77eec59 --- /dev/null +++ b/.github/scripts/check-milestone-branch.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# check-milestone-branch.sh +# +# Validates that a pull request's milestone is consistent with the branch the +# pull request targets. +# +# OVERVIEW +# -------- +# Milestones in this repository are named "..", optionally +# with a pre-release suffix (e.g. "7.0.3", "8.0.0-preview1"). Every milestone +# therefore maps to a candidate release branch: +# +# .. -> release/. +# +# Release branches and configured milestones determine where the work belongs: +# +# * The branch EXISTS -> that version has already forked off the default +# branch and is in servicing. Changes for it go to release/.. +# +# * The branch DOES NOT exist, and this is the earliest configured milestone +# series without a release branch -> that version is in development on the +# default branch. Changes for it go to the default branch. +# +# * Any other series -> the default branch carries exactly one development +# line, so a later series is not active yet and an earlier series is no +# longer in development. Neither may target the default branch. +# +# This rule is self-maintaining: no hard-coded version list needs updating when +# a new release branch is cut. +# +# VALIDATION MATRIX +# ----------------- +# Target branch Release branch exists? Result +# ----------------------- ----------------------- ---------------------- +# release/. n/a (it is the target) pass +# another release/* n/a fail (mismatch) +# default branch no, active line pass +# default branch no, later configured line fail (not active yet) +# default branch no, earlier configured line fail (no longer in development) +# default branch no active line configured fail (milestone missing) +# default branch yes fail (needs servicing branch) +# anything else n/a skipped (integration branch) +# +# Pull requests into long-lived integration branches (e.g. "dev/paul/foo") are +# skipped, because the milestone is enforced when that branch is merged into +# the default branch or a release branch. +# +# Milestones that don't parse as ".." are skipped with a +# notice rather than failing the build. +# +# REQUIRED ENVIRONMENT VARIABLES +# ------------------------------ +# MILESTONE_TITLE The PR's milestone title (e.g. "7.0.3"). +# BASE_REF The branch the PR targets (e.g. "main", "release/7.0"). +# DEFAULT_BRANCH The repository's default branch (e.g. "main"). +# GITHUB_REPOSITORY Owner/repo (e.g. "dotnet/SqlClient"). Set automatically by Actions. +# GH_TOKEN GitHub token for API calls (gh CLI auth). +# +# OUTPUTS +# ------- +# Emits ::notice:: on success/skip and ::error:: on failure. +# Exits 0 when the milestone and target branch agree (or the check is +# skipped), and 1 when they conflict. +# +# USAGE +# Called from the check-milestone.yml workflow. Can also be run locally: +# +# export MILESTONE_TITLE="7.0.3" +# export BASE_REF="main" +# export DEFAULT_BRANCH="main" +# export GITHUB_REPOSITORY="dotnet/SqlClient" +# bash .github/scripts/check-milestone-branch.sh +# +################################################################################# +set -euo pipefail + +# -- Runtime help ------------------------------------------------------------- +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + # Print the header comment block (between the license banner and the + # closing banner), stripping the leading '# ' prefix. + awk '/^#{2,}$/ { n++; next } n == 2 { sub(/^# ?/, ""); print }' "$0" + exit 0 +fi + +# -- Input validation --------------------------------------------------------- +: "${MILESTONE_TITLE:?MILESTONE_TITLE environment variable is required}" +: "${BASE_REF:?BASE_REF environment variable is required}" +: "${DEFAULT_BRANCH:?DEFAULT_BRANCH environment variable is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY environment variable is required}" + +# -- Derive the candidate release branch from the milestone ------------------- +# Accepts "X.Y.Z" with an optional pre-release/build suffix, e.g. "8.0.0-preview1". +if [[ "${MILESTONE_TITLE}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)([-+].*)?$ ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" +else + echo "::notice::Milestone '${MILESTONE_TITLE}' is not in 'major.minor.patch' form; skipping the target branch check." + exit 0 +fi + +RELEASE_BRANCH="release/${MAJOR}.${MINOR}" + +# -- Skip integration branches ------------------------------------------------ +# Only the default branch and release branches carry milestone semantics. +if [[ "${BASE_REF}" != "${DEFAULT_BRANCH}" && "${BASE_REF}" != release/* ]]; then + echo "::notice::PR targets integration branch '${BASE_REF}'; skipping the milestone/branch check." + exit 0 +fi + +# -- Validate a release branch target ----------------------------------------- +# The target branch itself proves which version is being serviced, so no branch +# listing is needed here. +if [[ "${BASE_REF}" == release/* ]]; then + if [[ "${BASE_REF}" != "${RELEASE_BRANCH}" ]]; then + echo "::error::Milestone '${MILESTONE_TITLE}' belongs to '${RELEASE_BRANCH}', but this PR targets '${BASE_REF}'. Retarget the PR or assign the milestone that matches '${BASE_REF}'." + exit 1 + fi + + echo "::notice::Milestone '${MILESTONE_TITLE}' matches target branch '${BASE_REF}'." + exit 0 +fi + +# -- Validate a default branch target ----------------------------------------- +# 'matching-refs' returns only refs under the given prefix, so this is a single +# cheap call regardless of how many topic branches the repository has. +if ! RELEASE_REFS=$(gh api "repos/${GITHUB_REPOSITORY}/git/matching-refs/heads/release/" \ + --jq '.[].ref' 2>&1); then + echo "::error::Unable to list release branches for '${GITHUB_REPOSITORY}': ${RELEASE_REFS}" + exit 1 +fi + +if grep -qxF "refs/heads/${RELEASE_BRANCH}" <<< "${RELEASE_REFS}"; then + echo "::error::Milestone '${MILESTONE_TITLE}' is a servicing release owned by '${RELEASE_BRANCH}', but this PR targets '${DEFAULT_BRANCH}'. Either retarget the PR to '${RELEASE_BRANCH}', or assign an in-development milestone and add the 'Hotfix ${MAJOR}.${MINOR}.${PATCH}' label so the change is cherry-picked after merge." + exit 1 +fi + +if ! MILESTONES=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/milestones?state=all&per_page=100" \ + --jq '.[].title' 2>&1); then + echo "::error::Unable to list milestones for '${GITHUB_REPOSITORY}': ${MILESTONES}" + exit 1 +fi + +ACTIVE_MAJOR="" +ACTIVE_MINOR="" +LATEST_RELEASE_MAJOR="" +LATEST_RELEASE_MINOR="" +while IFS= read -r release_ref; do + if [[ ! "${release_ref}" =~ ^refs/heads/release/([0-9]+)\.([0-9]+)$ ]]; then + continue + fi + + release_major="${BASH_REMATCH[1]}" + release_minor="${BASH_REMATCH[2]}" + if [[ -z "${LATEST_RELEASE_MAJOR}" ]] || + (( 10#${release_major} > 10#${LATEST_RELEASE_MAJOR} )) || + (( 10#${release_major} == 10#${LATEST_RELEASE_MAJOR} && 10#${release_minor} > 10#${LATEST_RELEASE_MINOR} )); then + LATEST_RELEASE_MAJOR="${release_major}" + LATEST_RELEASE_MINOR="${release_minor}" + fi +done <<< "${RELEASE_REFS}" + +while IFS= read -r milestone; do + if [[ ! "${milestone}" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)([-+].*)?$ ]]; then + continue + fi + + candidate_major="${BASH_REMATCH[1]}" + candidate_minor="${BASH_REMATCH[2]}" + candidate_branch="release/${candidate_major}.${candidate_minor}" + if grep -qxF "refs/heads/${candidate_branch}" <<< "${RELEASE_REFS}"; then + continue + fi + + if [[ -n "${LATEST_RELEASE_MAJOR}" ]] && + { (( 10#${candidate_major} < 10#${LATEST_RELEASE_MAJOR} )) || + (( 10#${candidate_major} == 10#${LATEST_RELEASE_MAJOR} && 10#${candidate_minor} <= 10#${LATEST_RELEASE_MINOR} )); }; then + continue + fi + + if [[ -z "${ACTIVE_MAJOR}" ]] || + (( 10#${candidate_major} < 10#${ACTIVE_MAJOR} )) || + (( 10#${candidate_major} == 10#${ACTIVE_MAJOR} && 10#${candidate_minor} < 10#${ACTIVE_MINOR} )); then + ACTIVE_MAJOR="${candidate_major}" + ACTIVE_MINOR="${candidate_minor}" + fi +done <<< "${MILESTONES}" + +if [[ -z "${ACTIVE_MAJOR}" ]]; then + echo "::error::No configured milestone series is newer than the newest release branch, so no development line is active on '${DEFAULT_BRANCH}'. Create the milestone for the next version before targeting '${DEFAULT_BRANCH}' with '${MILESTONE_TITLE}'." + exit 1 +fi + +if (( 10#${MAJOR} != 10#${ACTIVE_MAJOR} || 10#${MINOR} != 10#${ACTIVE_MINOR} )); then + if (( 10#${MAJOR} > 10#${ACTIVE_MAJOR} || + (10#${MAJOR} == 10#${ACTIVE_MAJOR} && 10#${MINOR} > 10#${ACTIVE_MINOR}) )); then + echo "::error::Milestone '${MILESTONE_TITLE}' is for a later development line, but the ${ACTIVE_MAJOR}.${ACTIVE_MINOR} milestone series remains active on '${DEFAULT_BRANCH}' until 'release/${ACTIVE_MAJOR}.${ACTIVE_MINOR}' is cut. Assign a milestone from the active ${ACTIVE_MAJOR}.${ACTIVE_MINOR} line." + else + echo "::error::Milestone '${MILESTONE_TITLE}' is for the ${MAJOR}.${MINOR} line, which is no longer in development on '${DEFAULT_BRANCH}'; the active line is ${ACTIVE_MAJOR}.${ACTIVE_MINOR}. Assign a milestone from the active ${ACTIVE_MAJOR}.${ACTIVE_MINOR} line." + fi + exit 1 +fi + +echo "::notice::Milestone '${MILESTONE_TITLE}' is still in development (no '${RELEASE_BRANCH}' branch); targeting '${DEFAULT_BRANCH}' is correct." diff --git a/.github/scripts/recheck-milestones-for-release-branch.sh b/.github/scripts/recheck-milestones-for-release-branch.sh new file mode 100755 index 0000000000..b48ceb4ad6 --- /dev/null +++ b/.github/scripts/recheck-milestones-for-release-branch.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# recheck-milestones-for-release-branch.sh +# +# Re-runs the milestone check for open pull requests whose result can change +# when a release branch is created. +# +# OVERVIEW +# -------- +# check-milestone-branch.sh decides where a milestone belongs by asking whether +# release/. exists and which configured milestone series is the +# earliest one without a release branch. Cutting a branch therefore moves its +# X.Y.* milestones into servicing and can make the next series active. +# +# Creating a branch emits no pull request activity, so an already-open PR keeps +# whatever result it last recorded. This script closes that gap by re-running +# every semantic-version-milestoned PR targeting the default branch. That covers +# both the newly serviced series and any later series whose eligibility changes. +# +# Re-running is enough because check-milestone-branch.sh queries the live list +# of release branches. The replayed event payload still carries the correct +# milestone and base branch, since the check re-runs on every 'milestoned' and +# 'edited' activity, so the newest run always reflects the current PR state. +# +# LIMITATION +# ---------- +# A re-run replays the original run's commit, which means it executes the +# workflow definition and script as they existed then. Only the release branch +# lookup is evaluated live. +# +# So a pull request whose most recent milestone check predates the arrival of +# the target-branch rule will replay the older check, pass it, and keep its +# stale result. This is transitional: any pull request with activity after the +# rule shipped has a run that contains it. It is not detected here, because +# distinguishing a stale replay costs an extra API call per pull request and +# the window closes on its own. After the first release branch is cut following +# a change to the check itself, review the affected pull requests by hand. +# +# REQUIRED ENVIRONMENT VARIABLES +# ------------------------------ +# RELEASE_BRANCH The branch that was just created (e.g. "release/7.1"). +# DEFAULT_BRANCH The repository's default branch (e.g. "main"). +# WORKFLOW_FILE Workflow file name to re-run (e.g. "check-milestone.yml"). +# GITHUB_REPOSITORY Owner/repo (e.g. "dotnet/SqlClient"). Set automatically by Actions. +# GH_TOKEN GitHub token for API calls. Needs 'actions: write'. +# +# OUTPUTS +# ------- +# Emits ::notice:: per PR re-run and ::warning:: for any PR that could not be +# re-run. Exits 1 if at least one re-run failed, so the failure is visible in +# the Actions UI; a maintainer can then re-run those checks by hand. +# +# USAGE +# Called from the recheck-milestones.yml workflow. Can also be run locally: +# +# export RELEASE_BRANCH="release/7.1" +# export DEFAULT_BRANCH="main" +# export WORKFLOW_FILE="check-milestone.yml" +# export GITHUB_REPOSITORY="dotnet/SqlClient" +# bash .github/scripts/recheck-milestones-for-release-branch.sh +# +################################################################################# +# 'set -e' is deliberately omitted: one PR failing to re-run must not abandon +# the rest. Failures are tracked explicitly and reported at the end. +set -uo pipefail + +# -- Runtime help ------------------------------------------------------------- +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + # Print the header comment block (between the license banner and the + # closing banner), stripping the leading '# ' prefix. + awk '/^#{2,}$/ { n++; next } n == 2 { sub(/^# ?/, ""); print }' "$0" + exit 0 +fi + +# -- Input validation --------------------------------------------------------- +: "${RELEASE_BRANCH:?RELEASE_BRANCH environment variable is required}" +: "${DEFAULT_BRANCH:?DEFAULT_BRANCH environment variable is required}" +: "${WORKFLOW_FILE:?WORKFLOW_FILE environment variable is required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY environment variable is required}" + +if [[ ! "${RELEASE_BRANCH}" =~ ^release/[0-9]+\.[0-9]+$ ]]; then + echo "::notice::'${RELEASE_BRANCH}' is not a 'release/.' branch; nothing to reconcile." + exit 0 +fi + +# -- Find open PRs whose result can change ------------------------------------- +if ! OPEN_PRS=$(gh pr list --repo "${GITHUB_REPOSITORY}" --base "${DEFAULT_BRANCH}" \ + --state open --limit 500 --json number,headRefOid,milestone \ + --jq '.[] | select(.milestone != null) | "\(.number) \(.headRefOid) \(.milestone.title)"' 2>&1); then + echo "::error::Unable to list open pull requests for '${GITHUB_REPOSITORY}': ${OPEN_PRS}" + exit 1 +fi + +FAILED=0 +MATCHED=0 + +while read -r NUMBER HEAD_SHA MILESTONE_TITLE; do + [[ -n "${NUMBER}" ]] || continue + + # Same milestone grammar as check-milestone-branch.sh. + [[ "${MILESTONE_TITLE}" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-+].*)?$ ]] || continue + + MATCHED=$((MATCHED + 1)) + + RUNS=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/runs?head_sha=${HEAD_SHA}&per_page=100" \ + 2>/dev/null) + + # One head commit can back PRs against several bases, so prefer the run that + # names this PR rather than just the newest run for the SHA. + RUN_ID=$(jq -r --argjson pr "${NUMBER}" \ + '([.workflow_runs[] | select(any(.pull_requests[]?; .number == $pr))] | first | .id) // empty' \ + <<< "${RUNS}" 2>/dev/null) + + # 'pull_requests' is empty for runs from forked repositories, so fall back to + # the newest run for the SHA when the association is unavailable. + if [[ -z "${RUN_ID}" ]]; then + RUN_ID=$(jq -r '.workflow_runs[0].id // empty' <<< "${RUNS}" 2>/dev/null) + fi + + if [[ -z "${RUN_ID}" ]]; then + echo "::warning::PR #${NUMBER} (milestone '${MILESTONE_TITLE}') has no milestone check run to re-run; re-check it manually." + FAILED=$((FAILED + 1)) + continue + fi + + if gh run rerun "${RUN_ID}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + echo "::notice::Re-ran the milestone check for PR #${NUMBER} (milestone '${MILESTONE_TITLE}', run ${RUN_ID})." + else + echo "::warning::Could not re-run the milestone check for PR #${NUMBER} (run ${RUN_ID}); re-check it manually." + FAILED=$((FAILED + 1)) + fi +done <<< "${OPEN_PRS}" + +if [[ "${MATCHED}" -eq 0 ]]; then + echo "::notice::No open PR targeting '${DEFAULT_BRANCH}' carries a semantic-version milestone." + exit 0 +fi + +if [[ "${FAILED}" -gt 0 ]]; then + echo "::error::${FAILED} of ${MATCHED} affected pull requests could not be re-checked automatically." + exit 1 +fi + +echo "::notice::Re-checked ${MATCHED} pull request(s) affected by '${RELEASE_BRANCH}'." diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index 9aac1c6c88..9a4ea68906 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -1,7 +1,9 @@ -# Cherry-Pick Workflow Tests +# GitHub Actions Script Tests This directory contains tests for the shell scripts used by the -[cherry-pick-hotfix](./../../../.github/workflows/cherry-pick-hotfix.yml) GitHub Actions workflow. +[cherry-pick-hotfix](./../../../.github/workflows/cherry-pick-hotfix.yml), +[check-milestone](./../../../.github/workflows/check-milestone.yml) and +[recheck-milestones](./../../../.github/workflows/recheck-milestones.yml) GitHub Actions workflows. These tests are intended to be run manually by developers when they are changing the associated scripts, and not as part of any CI runs. @@ -85,6 +87,8 @@ bats .github/scripts/tests/ ```bash bats .github/scripts/tests/extract-hotfix-versions.bats bats .github/scripts/tests/cherry-pick-to-release.bats +bats .github/scripts/tests/check-milestone-branch.bats +bats .github/scripts/tests/recheck-milestones-for-release-branch.bats ``` ### Run a specific test by name @@ -112,10 +116,12 @@ bats --formatter pretty .github/scripts/tests/ | ---- | ----- | ------ | | `extract-hotfix-versions.bats` | 18 | Label parsing, version extraction, matrix JSON output, edge cases (malformed labels, duplicates, `labeled` vs `closed` events) | | `cherry-pick-to-release.bats` | 15 | Branch derivation, already-applied detection, clean cherry-pick, conflict handling, milestone lookup, PR creation, duplicate skip logic | +| `check-milestone-branch.bats` | 26 | Milestone version parsing, state-independent active development-line selection, rejection of earlier and later series on the default branch, fail-closed handling when no series is active, release-branch derivation, default-branch vs release-branch validation, integration-branch and non-semver skips, API invocation assertions, API failure handling | +| `recheck-milestones-for-release-branch.bats` | 17 | Release-branch name parsing, matching open PRs by milestone, run lookup by head SHA and PR association, fork fallback, re-run invocation, and failure reporting | ## How the Tests Work -Both test files use the same general approach: +All test files use the same general approach: 1. **`setup()`** creates a temporary directory and populates it with mock `git` and `gh` executables — simple shell scripts that echo predetermined responses. Environment variables (`VERSION`, diff --git a/.github/scripts/tests/check-milestone-branch.bats b/.github/scripts/tests/check-milestone-branch.bats new file mode 100644 index 0000000000..813605a012 --- /dev/null +++ b/.github/scripts/tests/check-milestone-branch.bats @@ -0,0 +1,317 @@ +#!/usr/bin/env bats +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Tests for check-milestone-branch.sh +# +# Run with: bats .github/scripts/tests/check-milestone-branch.bats +# +# Dependencies: bats-core (https://github.com/bats-core/bats-core) +# +################################################################################# + +# Path to the script under test (relative to repo root). +SCRIPT=".github/scripts/check-milestone-branch.sh" + +# ── Helpers ────────────────────────────────────────────────────────────────── + +setup() { + STUB_DIR="$(mktemp -d)" + export PATH="${STUB_DIR}:${PATH}" + + # Defaults — individual tests override as needed. + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + export DEFAULT_BRANCH="main" + export GITHUB_REPOSITORY="dotnet/SqlClient" + export GH_TOKEN="fake-token" + export MOCK_MILESTONES=$'1.0.0\n2.0.1\n7.1.0\n8.0.0-preview1\n8.0.0-preview2\n8.0.0' + + mock_release_branches "release/6.1" "release/7.0" +} + +teardown() { + rm -rf "${STUB_DIR}" +} + +# Install a 'gh' mock that reports the given release branches. +mock_release_branches() { + local refs="" + local branch + for branch in "$@"; do + refs+="refs/heads/${branch}"$'\n' + done + + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +if [[ "\$*" == *"/milestones"* ]]; then + printf '%s' "\${MOCK_MILESTONES}" +else + printf '%s' '${refs}' +fi +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock that fails, simulating an API error. +mock_gh_failure() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +echo "HTTP 403: rate limit exceeded" >&2 +exit 1 +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock that lists branches but fails when milestones are queried. +mock_milestone_failure() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +if [[ "\$*" == *"/milestones"* ]]; then + echo "HTTP 403: resource not accessible by integration" >&2 + exit 1 +fi +printf '%s' 'refs/heads/release/6.1 +refs/heads/release/7.0 +' +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# ── --help flag ────────────────────────────────────────────────────────────── + +@test "prints help text with --help" { + run bash "${SCRIPT}" --help + [ "$status" -eq 0 ] + [[ "$output" == *"VALIDATION MATRIX"* ]] + [[ "$output" == *"REQUIRED ENVIRONMENT VARIABLES"* ]] +} + +@test "prints help text with -h" { + run bash "${SCRIPT}" -h + [ "$status" -eq 0 ] + [[ "$output" == *"VALIDATION MATRIX"* ]] +} + +# ── Input validation ───────────────────────────────────────────────────────── + +@test "fails when MILESTONE_TITLE is unset" { + unset MILESTONE_TITLE + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"MILESTONE_TITLE"* ]] +} + +@test "fails when BASE_REF is unset" { + unset BASE_REF + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"BASE_REF"* ]] +} + +@test "fails when DEFAULT_BRANCH is unset" { + unset DEFAULT_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"DEFAULT_BRANCH"* ]] +} + +@test "fails when GITHUB_REPOSITORY is unset" { + unset GITHUB_REPOSITORY + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"GITHUB_REPOSITORY"* ]] +} + +# ── Default branch targets ─────────────────────────────────────────────────── + +@test "passes when in-development milestone targets the default branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"::notice::"* ]] + [[ "$output" == *"still in development"* ]] +} + +@test "fails when a later milestone targets the default branch before the active release branch is cut" { + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"7.1 milestone series"* ]] + [[ "$output" == *"release/7.1"* ]] +} + +@test "closed milestone state does not change the active development line" { + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"7.1 milestone series"* ]] + [[ "$output" != *"1.0.0"* ]] + grep -qF "milestones?state=all" "${STUB_DIR}/gh.log" +} + +@test "fails when an earlier milestone series targets the default branch" { + export MILESTONE_TITLE="1.0.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"no longer in development"* ]] + [[ "$output" == *"active line is 7.1"* ]] +} + +@test "fails when no configured milestone series is active" { + mock_release_branches "release/7.0" + export MOCK_MILESTONES=$'1.0.0' + export MILESTONE_TITLE="1.0.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"no development line is active"* ]] +} + +@test "passes when a later milestone targets the default branch after the active release branch is cut" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="8.0.0-preview1" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"still in development"* ]] +} + +@test "fails when a serviced milestone targets the default branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"::error::"* ]] + [[ "$output" == *"release/7.0"* ]] + [[ "$output" == *"Hotfix 7.0.3"* ]] +} + +@test "honours a non-'main' default branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="master" + export DEFAULT_BRANCH="master" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"targeting 'master' is correct"* ]] +} + +# ── API invocation ─────────────────────────────────────────────────────────── + +@test "queries the release refs endpoint with the expected arguments" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: api repos/dotnet/SqlClient/git/matching-refs/heads/release/ --jq .[].ref" "${STUB_DIR}/gh.log" + grep -qF "GH: api --paginate repos/dotnet/SqlClient/milestones?state=all&per_page=100 --jq .[].title" "${STUB_DIR}/gh.log" +} + +@test "does not call the API when the PR targets a release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [ ! -f "${STUB_DIR}/gh.log" ] +} + +# ── Release branch targets ─────────────────────────────────────────────────── + +@test "passes when the milestone matches the target release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"matches target branch 'release/7.0'"* ]] +} + +@test "fails when the milestone belongs to a different release branch" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="release/6.1" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"::error::"* ]] + [[ "$output" == *"belongs to 'release/7.0'"* ]] +} + +@test "fails when an in-development milestone targets a release branch" { + export MILESTONE_TITLE="7.1.0" + export BASE_REF="release/7.0" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"belongs to 'release/7.1'"* ]] +} + +@test "passes when a newly cut release branch matches the milestone" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="7.1.0" + export BASE_REF="release/7.1" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"matches target branch 'release/7.1'"* ]] +} + +@test "fails on the default branch once the release branch is cut" { + mock_release_branches "release/6.1" "release/7.0" "release/7.1" + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"release/7.1"* ]] +} + +# ── Skipped cases ──────────────────────────────────────────────────────────── + +@test "skips integration branch targets" { + export MILESTONE_TITLE="7.0.3" + export BASE_REF="dev/paul/some-feature" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"integration branch"* ]] +} + +@test "skips milestones that are not major.minor.patch" { + export MILESTONE_TITLE="vNext" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"not in 'major.minor.patch' form"* ]] +} + +@test "skips two-part milestone titles" { + export MILESTONE_TITLE="1.0 Hotfix 2" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"not in 'major.minor.patch' form"* ]] +} + +# ── API failures ───────────────────────────────────────────────────────────── + +@test "fails when the branch listing API call fails" { + mock_gh_failure + export MILESTONE_TITLE="7.0.3" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list release branches"* ]] +} + +@test "fails when the milestone listing API call fails" { + mock_milestone_failure + export MILESTONE_TITLE="7.1.0" + export BASE_REF="main" + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list milestones"* ]] +} diff --git a/.github/scripts/tests/recheck-milestones-for-release-branch.bats b/.github/scripts/tests/recheck-milestones-for-release-branch.bats new file mode 100644 index 0000000000..0d48388f33 --- /dev/null +++ b/.github/scripts/tests/recheck-milestones-for-release-branch.bats @@ -0,0 +1,228 @@ +#!/usr/bin/env bats +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Tests for recheck-milestones-for-release-branch.sh +# +# Run with: bats .github/scripts/tests/recheck-milestones-for-release-branch.bats +# +# Dependencies: bats-core (https://github.com/bats-core/bats-core) +# +################################################################################# + +# Path to the script under test (relative to repo root). +SCRIPT=".github/scripts/recheck-milestones-for-release-branch.sh" + +# ── Helpers ────────────────────────────────────────────────────────────────── + +setup() { + STUB_DIR="$(mktemp -d)" + export PATH="${STUB_DIR}:${PATH}" + + # Defaults — individual tests override as needed. + export RELEASE_BRANCH="release/7.1" + export DEFAULT_BRANCH="main" + export WORKFLOW_FILE="check-milestone.yml" + export GITHUB_REPOSITORY="dotnet/SqlClient" + export GH_TOKEN="fake-token" + + # Open PRs as " ", one per line. + mock_gh "100 aaa111 7.1.0 +101 bbb222 8.0.0-preview1 +102 ccc333 7.1.0-preview3" + + # Default: one run per head SHA, correctly associated with its PR. + set_runs aaa111 '{"workflow_runs":[{"id":"run-aaa111","pull_requests":[{"number":100}]}]}' + set_runs bbb222 '{"workflow_runs":[{"id":"run-bbb222","pull_requests":[{"number":101}]}]}' + set_runs ccc333 '{"workflow_runs":[{"id":"run-ccc333","pull_requests":[{"number":102}]}]}' +} + +teardown() { + rm -rf "${STUB_DIR}" +} + +# Register the workflow-runs response for a given head SHA. +set_runs() { + printf '%s' "$2" > "${STUB_DIR}/runs-$1.json" +} + +# Install a 'gh' mock. $1 is the 'pr list' output; 'api' serves the JSON +# registered by set_runs, and 'run rerun' succeeds unless RERUN_FAILS is set. +mock_gh() { + cat > "${STUB_DIR}/gh" <> "${STUB_DIR}/gh.log" +case "\$1" in + pr) + printf '%s\n' '${1}' + ;; + api) + sha="\$(sed -n 's/.*head_sha=\([^&]*\).*/\1/p' <<< "\$2")" + if [[ -n "\${NO_RUN_FOUND:-}" || ! -f "${STUB_DIR}/runs-\${sha}.json" ]]; then + echo '{"workflow_runs":[]}' + else + cat "${STUB_DIR}/runs-\${sha}.json" + fi + ;; + run) + [[ -z "\${RERUN_FAILS:-}" ]] || exit 1 + ;; +esac +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# Install a 'gh' mock whose 'pr list' call fails. +mock_pr_list_failure() { + cat > "${STUB_DIR}/gh" <<'MOCK' +#!/usr/bin/env bash +echo "HTTP 403: rate limit exceeded" >&2 +exit 1 +MOCK + chmod +x "${STUB_DIR}/gh" +} + +# ── --help flag ────────────────────────────────────────────────────────────── + +@test "prints help text with --help" { + run bash "${SCRIPT}" --help + [ "$status" -eq 0 ] + [[ "$output" == *"REQUIRED ENVIRONMENT VARIABLES"* ]] +} + +# ── Input validation ───────────────────────────────────────────────────────── + +@test "fails when RELEASE_BRANCH is unset" { + unset RELEASE_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"RELEASE_BRANCH"* ]] +} + +@test "fails when DEFAULT_BRANCH is unset" { + unset DEFAULT_BRANCH + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"DEFAULT_BRANCH"* ]] +} + +@test "fails when WORKFLOW_FILE is unset" { + unset WORKFLOW_FILE + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"WORKFLOW_FILE"* ]] +} + +@test "fails when GITHUB_REPOSITORY is unset" { + unset GITHUB_REPOSITORY + run bash "${SCRIPT}" + [ "$status" -ne 0 ] + [[ "$output" == *"GITHUB_REPOSITORY"* ]] +} + +# ── Branch name parsing ────────────────────────────────────────────────────── + +@test "skips branches that are not release/." { + export RELEASE_BRANCH="dev/paul/some-feature" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"nothing to reconcile"* ]] + [ ! -f "${STUB_DIR}/gh.log" ] +} + +@test "skips a release branch with a patch component" { + export RELEASE_BRANCH="release/7.1.0" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"nothing to reconcile"* ]] +} + +# ── Matching and re-running ────────────────────────────────────────────────── + +@test "re-runs all semver-milestoned PRs affected by active-line transitions" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"PR #100"* ]] + [[ "$output" == *"PR #102"* ]] + [[ "$output" == *"PR #101"* ]] + [[ "$output" == *"Re-checked 3 pull request(s)"* ]] +} + +@test "queries open PRs against the default branch" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: pr list --repo dotnet/SqlClient --base main --state open" "${STUB_DIR}/gh.log" +} + +@test "looks up the run by the PR head sha and re-runs it" { + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "head_sha=aaa111" "${STUB_DIR}/gh.log" + grep -qF "GH: run rerun run-aaa111 --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" +} + +@test "picks the run belonging to this PR when a head sha backs several PRs" { + # The newest run for the SHA belongs to a different PR against another base. + set_runs aaa111 '{"workflow_runs":[ + {"id":"run-other","pull_requests":[{"number":999}]}, + {"id":"run-aaa111","pull_requests":[{"number":100}]} + ]}' + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + grep -qF "GH: run rerun run-aaa111 --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" + ! grep -qF "run rerun run-other" "${STUB_DIR}/gh.log" +} + +@test "falls back to the newest run when the PR association is missing" { + # Runs from forked repositories carry an empty 'pull_requests' array. + set_runs aaa111 '{"workflow_runs":[ + {"id":"run-newest","pull_requests":[]}, + {"id":"run-older","pull_requests":[]} + ]}' + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"PR #100"* ]] + grep -qF "GH: run rerun run-newest --repo dotnet/SqlClient" "${STUB_DIR}/gh.log" +} + +@test "reports when no open PR carries a matching milestone" { + export RELEASE_BRANCH="release/6.1" + mock_gh "200 ddd444 vNext" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"No open PR targeting 'main' carries a semantic-version milestone"* ]] +} + +@test "ignores PRs whose milestone is not major.minor.patch" { + mock_gh "200 ddd444 vNext" + run bash "${SCRIPT}" + [ "$status" -eq 0 ] + [[ "$output" == *"No open PR"* ]] +} + +# ── Failure handling ───────────────────────────────────────────────────────── + +@test "warns and fails when a PR has no run to re-run" { + export NO_RUN_FOUND=1 + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"has no milestone check run to re-run"* ]] + [[ "$output" == *"3 of 3 affected pull requests"* ]] +} + +@test "warns and fails when a re-run cannot be started" { + export RERUN_FAILS=1 + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Could not re-run the milestone check for PR #100"* ]] +} + +@test "fails when the PR listing API call fails" { + mock_pr_list_failure + run bash "${SCRIPT}" + [ "$status" -eq 1 ] + [[ "$output" == *"Unable to list open pull requests"* ]] +} diff --git a/.github/workflows/auto-assign-pr.yml b/.github/workflows/auto-assign-pr.yml index ba5654969f..aab463f2e2 100644 --- a/.github/workflows/auto-assign-pr.yml +++ b/.github/workflows/auto-assign-pr.yml @@ -23,7 +23,7 @@ jobs: PR_REVIEWER_POOL: ${{ vars.PR_REVIEWER_POOL }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: sparse-checkout: | .github/scripts/auto-assign-pr.js diff --git a/.github/workflows/check-milestone.yml b/.github/workflows/check-milestone.yml index da4681fd7b..8d64ae5f13 100644 --- a/.github/workflows/check-milestone.yml +++ b/.github/workflows/check-milestone.yml @@ -1,14 +1,45 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Check Milestone +# +# Validates that every pull request has an open milestone assigned, and that +# the milestone is consistent with the branch the PR targets. +# +# Milestones map to release branches by major.minor: +# +# * "7.0.3" -> release/7.0 exists -> the PR must target release/7.0. +# * "7.1.0" -> release/7.1 does not exist and it is the earliest configured +# unbranched series -> the PR must target main. +# * "8.0.0-preview1" -> release/8.0 does not exist, but 7.1 is still the +# active unbranched series -> the PR cannot target main yet. +# +# See .github/scripts/check-milestone-branch.sh for the full rule set. +# +# Cutting release/X.Y flips the expected target for X.Y.* milestones but emits +# no pull request activity. recheck-milestones.yml reconciles the already open +# PRs that the new branch invalidates. +# +################################################################################# + name: Check Milestone on: pull_request: - types: [opened, edited, synchronize, milestoned, demilestoned] + # The 'edited' type covers base branch changes, so retargeting a PR (manually, + # or automatically when a stacked PR's parent merges) re-runs this check. + types: [opened, reopened, edited, synchronize, milestoned, demilestoned] jobs: check-milestone: name: Validate milestone runs-on: ubuntu-latest permissions: + contents: read + issues: read pull-requests: read steps: - name: Check milestone is set @@ -22,3 +53,21 @@ jobs: run: | echo "::error::Milestone '${{ github.event.pull_request.milestone.title }}' is ${{ github.event.pull_request.milestone.state }}. Please assign an open milestone." exit 1 + + - name: Checkout scripts + if: github.event.pull_request.milestone != null + uses: actions/checkout@v6 + with: + # Only the scripts directory is needed; skip full history. + sparse-checkout: .github/scripts + sparse-checkout-cone-mode: false + + - name: Check milestone matches target branch + if: github.event.pull_request.milestone != null + env: + # Pass PR data via env to avoid script injection from milestone text. + MILESTONE_TITLE: ${{ github.event.pull_request.milestone.title }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/check-milestone-branch.sh" diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml index 886e8e2e63..d1ef3b2e67 100644 --- a/.github/workflows/issue-triage.lock.yml +++ b/.github/workflows/issue-triage.lock.yml @@ -1,15 +1,15 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"c9a288bea873cc74dd33e81e0f5784e167d990ee0e2645c1722106b45ae1f77d","body_hash":"99b21e9ba167d3bfbcfaeb3b04c515fc42b56679f0ee175c119bc5843f034d31","compiler_version":"v0.80.9","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.63"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"8c7d04ebf1ece56cd381446125da3e0f6896294a","version":"v0.80.9"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.7","digest":"sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7","digest":"sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.7","digest":"sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.27","digest":"sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.27@sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.4.0","digest":"sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036","pinned_image":"ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036"}]} -# This file was automatically generated by gh-aw (v0.80.9). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8b81a4840372af279438b0250d6ae7168fa2e0a2f82ae8cd52d714f18a83d425","body_hash":"99b21e9ba167d3bfbcfaeb3b04c515fc42b56679f0ee175c119bc5843f034d31","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","missing_data","missing_tool","noop","remove_labels"]}]} +# This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # -# ___ _ _ -# / _ \ | | (_) -# | |_| | __ _ ___ _ __ | |_ _ ___ +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ # | _ |/ _` |/ _ \ '_ \| __| |/ __| -# | | | | (_| | __/ | | | |_| | (__ +# | | | | (_| | __/ | | | |_| | (__ # \_| |_/\__, |\___|_| |_|\__|_|\___| # __/ | -# _ _ |___/ +# _ _ |___/ # | | | | / _| | # | | | | ___ _ __ _ __| |_| | _____ ____ # | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| @@ -26,45 +26,55 @@ # # Secrets used: # - COPILOT_GITHUB_TOKEN +# - GH_AW_DEFAULT_OTLP_HEADERS # - GH_AW_GITHUB_MCP_SERVER_TOKEN # - GH_AW_GITHUB_TOKEN # - GITHUB_TOKEN # # Custom actions used: -# - actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 -# - actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 -# - actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 +# - actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 +# - actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 +# - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 -# - actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 +# - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 -# - github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 +# - github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 # # Container images used: -# - ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c -# - ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6 -# - ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96 -# - ghcr.io/github/gh-aw-mcpg:v0.3.27@sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7 -# - ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b -# - ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036 +# - ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 +# - ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 +# - ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f +# - ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e +# - ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e +# - ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699 name: "SqlClient Issue Auto-Triage" on: issue_comment: types: - - created + - created issues: types: - - opened - # roles: all # Roles processed as role check in pre-activation job + - opened +# roles: all # Roles processed as role check in pre-activation job permissions: {} concurrency: group: "gh-aw-${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}" + queue: max run-name: "SqlClient Issue Auto-Triage" +env: + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.GH_AW_DEFAULT_OTLP_ENDPOINT }} + OTEL_SERVICE_NAME: gh-aw.issue-triage + OTEL_RESOURCE_ATTRIBUTES: 'gh-aw.workflow.name=SqlClient%20Issue%20Auto-Triage,gh-aw.repository=${{ github.repository }},gh-aw.run.id=${{ github.run_id }},github.run_id=${{ github.run_id }},gh-aw.engine.id=copilot' + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.GH_AW_DEFAULT_OTLP_HEADERS }} + GH_AW_OTLP_ENDPOINTS: '[{"url":"${{ vars.GH_AW_DEFAULT_OTLP_ENDPOINT }}","headers":"${{ secrets.GH_AW_DEFAULT_OTLP_HEADERS }}"}]' + GH_AW_OTLP_IF_MISSING: ignore + jobs: activation: if: > @@ -87,16 +97,19 @@ jobs: contents: read env: GH_AW_MAX_DAILY_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_DAILY_AI_CREDITS || '5000' }} + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} outputs: body: ${{ steps.sanitized.outputs.body }} comment_id: "" comment_repo: "" daily_ai_credits_exceeded: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_exceeded == 'true' }} + daily_ai_credits_guardrail_status: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_guardrail_status || '' }} daily_ai_credits_threshold: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_threshold || '' }} daily_ai_credits_total_effective_tokens: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_total_effective_tokens || '' }} engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} model: ${{ steps.generate_aw_info.outputs.model }} + oauth_token_check_failed: ${{ steps.check-oauth-tokens.outputs.oauth_token_check_failed == 'true' }} setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} @@ -106,7 +119,7 @@ jobs: steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -114,40 +127,45 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" - name: Generate agentic run info id: generate_aw_info env: GH_AW_INFO_ENGINE_ID: "copilot" GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AGENT_VERSION: "1.0.63" - GH_AW_INFO_CLI_VERSION: "v0.80.9" + GH_AW_INFO_MODEL: "auto" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AGENT_VERSION: "1.0.80" + GH_AW_INFO_CLI_VERSION: "v0.88.2" GH_AW_INFO_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_INFO_EXPERIMENTAL: "false" GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" GH_AW_INFO_STAGED: "false" GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' GH_AW_INFO_FIREWALL_ENABLED: "true" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_AWMG_VERSION: "" GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_INFO_AGENT_RUNTIME: "" GH_AW_COMPILED_STRICT: "true" uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_aw_info.cjs'); + const { main } = require(path.join(actionsDir, 'generate_aw_info.cjs')); await main(core, context); - name: Restore daily AIC usage cache id: restore-daily-aic-cache if: ${{ env.GH_AW_MAX_DAILY_AI_CREDITS != '' }} continue-on-error: true - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: key: agentic-workflow-usage-issuetriage-${{ github.run_id }} restore-keys: agentic-workflow-usage-issuetriage- @@ -163,9 +181,11 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/restore_aic_usage_cache_fallback.cjs'); + const { main } = require(path.join(actionsDir, 'restore_aic_usage_cache_fallback.cjs')); await main(); - name: Check daily workflow token guardrail id: daily-effective-workflow-guardrail @@ -183,32 +203,38 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_daily_aic_workflow_guardrail.cjs'); + const { main } = require(path.join(actionsDir, 'check_daily_aic_workflow_guardrail.cjs')); await main(); + - name: Check for OAuth tokens + id: check-oauth-tokens + run: bash "${RUNNER_TEMP}/gh-aw/actions/check_oauth_tokens.sh" + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} - name: Checkout .github and .agents folders - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: | .github .agents - .antigravity .claude .codex - .crush .gemini - .opencode .pi sparse-checkout-cone-mode: true fetch-depth: 1 - name: Save agent config folders for base branch restoration env: - GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" - GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" + GH_AW_AGENT_FOLDERS: ".agents .github" + GH_AW_AGENT_FILES: "AGENTS.md" + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/save_base_github_folders.sh" - name: Check workflow lock file id: check-lock-file uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -217,35 +243,47 @@ jobs: GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_workflow_timestamp_api.cjs'); + const { main } = require(path.join(actionsDir, 'check_workflow_timestamp_api.cjs')); await main(); - name: Check compile-agentic version uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_COMPILED_VERSION: "v0.80.9" + GH_AW_COMPILED_VERSION: "v0.88.2" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/check_version_updates.cjs'); + const { main } = require(path.join(actionsDir, 'check_version_updates.cjs')); await main(); - name: Compute current body text id: sanitized uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/compute_text.cjs'); + const { main } = require(path.join(actionsDir, 'compute_text.cjs')); await main(); + - name: Log runtime features + if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh" - name: Create prompt with built-in context + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ACTIONS_DIR: ${{ runner.temp }}/gh-aw/actions + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_PROMPT_CONFIG: "{\"items\":[{\"content_env\":\"GH_AW_PROMPT_CONTENT_0000\"},{\"file\":\"xpia.md\"},{\"file\":\"temp_folder_prompt.md\"},{\"file\":\"markdown.md\"},{\"file\":\"safe_outputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0001\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0002\"},{\"file\":\"mcp_cli_tools_with_safeoutputs_prompt.md\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0003\"},{\"file\":\"github_mcp_tools_with_safeoutputs_prompt.md\"},{\"file\":\"pr_context_prompt.md\",\"condition_env\":\"GH_AW_INCLUDE_PR_CONTEXT\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0004\"},{\"content_env\":\"GH_AW_PROMPT_CONTENT_0005\"}]}" GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} @@ -254,78 +292,36 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_IS_PR_COMMENT: ${{ github.event.issue.pull_request && 'true' || '' }} - # poutine:ignore untrusted_checkout_exec - run: | - bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" - { - cat << 'GH_AW_PROMPT_09a4b5a8a8b33a81_EOF' - - GH_AW_PROMPT_09a4b5a8a8b33a81_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" - cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_09a4b5a8a8b33a81_EOF' - - Tools: add_comment, add_labels, remove_labels, missing_tool, missing_data, noop - - GH_AW_PROMPT_09a4b5a8a8b33a81_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_09a4b5a8a8b33a81_EOF' - - The following GitHub context information is available for this workflow: - {{#if github.actor}} - - **actor**: __GH_AW_GITHUB_ACTOR__ - {{/if}} - {{#if github.repository}} - - **repository**: __GH_AW_GITHUB_REPOSITORY__ - {{/if}} - {{#if github.workspace}} - - **workspace**: __GH_AW_GITHUB_WORKSPACE__ - {{/if}} - {{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}} - - **issue-number**: #__GH_AW_EXPR_802A9F6A__ - {{/if}} - {{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}} - - **discussion-number**: #__GH_AW_EXPR_1A3A194A__ - {{/if}} - {{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}} - - **pull-request-number**: #__GH_AW_EXPR_463A214A__ - {{/if}} - {{#if github.event.comment.id || github.aw.context.comment_id}} - - **comment-id**: __GH_AW_EXPR_FF1D34CE__ - {{/if}} - {{#if github.run_id}} - - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ - {{/if}} - - - GH_AW_PROMPT_09a4b5a8a8b33a81_EOF - cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - if [ "$GITHUB_EVENT_NAME" = "issue_comment" ] && [ -n "$GH_AW_IS_PR_COMMENT" ] || [ "$GITHUB_EVENT_NAME" = "pull_request_review_comment" ] || [ "$GITHUB_EVENT_NAME" = "pull_request_review" ]; then - cat "${RUNNER_TEMP}/gh-aw/prompts/pr_context_prompt.md" - fi - cat << 'GH_AW_PROMPT_09a4b5a8a8b33a81_EOF' - - {{#runtime-import .github/workflows/issue-triage.md}} - GH_AW_PROMPT_09a4b5a8a8b33a81_EOF - } > "$GH_AW_PROMPT" + GH_AW_INCLUDE_PR_CONTEXT: ${{ (github.event_name == 'issue_comment' && github.event.issue.pull_request != null) || github.event_name == 'pull_request_review_comment' || github.event_name == 'pull_request_review' }} + GH_AW_PROMPT_CONTENT_0000: "\n" + GH_AW_PROMPT_CONTENT_0001: "\nTools: add_comment, add_labels, remove_labels, missing_tool, missing_data, noop\n" + GH_AW_PROMPT_CONTENT_0002: "\n" + GH_AW_PROMPT_CONTENT_0003: "\nThe following GitHub context information is available for this workflow:\n{{#if github.actor}}\n- **actor**: __GH_AW_GITHUB_ACTOR__\n{{/if}}\n{{#if github.repository}}\n- **repository**: __GH_AW_GITHUB_REPOSITORY__\n{{/if}}\n{{#if github.workspace}}\n- **workspace**: __GH_AW_GITHUB_WORKSPACE__\n{{/if}}\n{{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}\n- **issue-number**: #__GH_AW_EXPR_802A9F6A__\n{{/if}}\n{{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}\n- **discussion-number**: #__GH_AW_EXPR_1A3A194A__\n{{/if}}\n{{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}\n- **pull-request-number**: #__GH_AW_EXPR_463A214A__\n{{/if}}\n{{#if github.event.comment.id || github.aw.context.comment_id}}\n- **comment-id**: __GH_AW_EXPR_FF1D34CE__\n{{/if}}\n{{#if github.run_id}}\n- **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__\n{{/if}}\n\n\n" + GH_AW_PROMPT_CONTENT_0004: "\n" + GH_AW_PROMPT_CONTENT_0005: "{{#runtime-import .github/workflows/issue-triage.md}}\n" + with: + script: | + const { setupGlobals } = require(process.env.GH_AW_ACTIONS_DIR + '/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(process.env.GH_AW_ACTIONS_DIR + '/create_prompt.cjs'); + await main(core); - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_ENGINE_ID: "copilot" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/interpolate_prompt.cjs'); + const { main } = require(path.join(actionsDir, 'interpolate_prompt.cjs')); await main(); - name: Substitute placeholders uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} @@ -334,15 +330,17 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} - GH_AW_IS_PR_COMMENT: ${{ github.event.issue.pull_request && 'true' || '' }} - GH_AW_MCP_CLI_SERVERS_LIST: '- `safeoutputs` — run `safeoutputs --help` to see available tools' + GH_AW_INCLUDE_PR_CONTEXT: ${{ (github.event_name == 'issue_comment' && github.event.issue.pull_request != null) || github.event_name == 'pull_request_review_comment' || github.event_name == 'pull_request_review' }} + GH_AW_MCP_CLI_SERVERS_LIST: "- `github` — run `github --help` to see available tools\n- `safeoutputs` — run `safeoutputs --help` to see available tools" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - - const substitutePlaceholders = require('${{ runner.temp }}/gh-aw/actions/substitute_placeholders.cjs'); - + + const substitutePlaceholders = require(path.join(actionsDir, 'substitute_placeholders.cjs')); + // Call the substitution function return await substitutePlaceholders({ file: process.env.GH_AW_PROMPT, @@ -355,22 +353,26 @@ jobs: GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, - GH_AW_IS_PR_COMMENT: process.env.GH_AW_IS_PR_COMMENT, + GH_AW_INCLUDE_PR_CONTEXT: process.env.GH_AW_INCLUDE_PR_CONTEXT, GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST } }); - name: Validate prompt placeholders env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/validate_prompt_placeholders.sh" - name: Print prompt env: - GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - # poutine:ignore untrusted_checkout_exec - run: bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + GH_AW_PROMPT: ${{ runner.temp }}/gh-aw/aw-prompts/prompt.txt + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/print_prompt_summary.sh" + - name: Stage prompt files for artifact upload + run: | + mkdir -p /tmp/gh-aw/aw-prompts + cp -a "${RUNNER_TEMP}/gh-aw/aw-prompts/." /tmp/gh-aw/aw-prompts/ - name: Upload activation artifact - if: success() + if: success() || failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: activation @@ -397,12 +399,20 @@ jobs: contents: read issues: read pull-requests: read + timeout-minutes: 60 env: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GH_AW_ASSETS_ALLOWED_EXTS: "" GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_PR_HEAD_BASE_BRANCH: "" + GH_AW_PR_HEAD_BASE_PR_NUMBER: "" + GH_AW_PR_HEAD_BASE_REF: "" + GH_AW_PR_HEAD_BASE_REPO: "" + GH_AW_PR_HEAD_BASE_SHA: "" + GH_AW_PR_HEAD_REPO: "" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} GH_AW_WORKFLOW_ID_SANITIZED: issuetriage outputs: agentic_engine_timeout: ${{ steps.detect-agent-errors.outputs.agentic_engine_timeout || 'false' }} @@ -412,8 +422,13 @@ jobs: checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} has_patch: ${{ steps.collect_output.outputs.has_patch }} + http_400_response_error: ${{ steps.detect-agent-errors.outputs.http_400_response_error || 'false' }} inference_access_error: ${{ steps.detect-agent-errors.outputs.inference_access_error || 'false' }} + invocation_cap_exceeded: ${{ steps.detect-agent-errors.outputs.invocation_cap_exceeded || 'false' }} + max_cache_misses_exceeded: ${{ steps.detect-agent-errors.outputs.max_cache_misses_exceeded || 'false' }} mcp_policy_error: ${{ steps.detect-agent-errors.outputs.mcp_policy_error || 'false' }} + missing_model_pricing_error: ${{ steps.detect-agent-errors.outputs.missing_model_pricing_error || 'false' }} + missing_model_pricing_model_name: ${{ steps.detect-agent-errors.outputs.missing_model_pricing_model_name || '' }} model: ${{ needs.activation.outputs.model }} model_not_supported_error: ${{ steps.detect-agent-errors.outputs.model_not_supported_error || 'false' }} output: ${{ steps.collect_output.outputs.output }} @@ -421,11 +436,12 @@ jobs: setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} + shell_expansion_guard_rejected: ${{ steps.detect-agent-errors.outputs.shell_expansion_guard_rejected || 'false' }} unknown_model_ai_credits: ${{ steps.parse-mcp-gateway.outputs.unknown_model_ai_credits || 'false' }} steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -434,19 +450,28 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" - name: Set runtime paths id: set-runtime-paths + env: + GH_AW_RUNNER_TOOL_CACHE: ${{ runner.tool_cache }} run: | + if [ -z "${RUNNER_TOOL_CACHE:-}" ]; then + echo "RUNNER_TOOL_CACHE=${GH_AW_RUNNER_TOOL_CACHE}" >> "$GITHUB_ENV" + fi { echo "GH_AW_SAFE_OUTPUTS=${RUNNER_TEMP}/gh-aw/safeoutputs/outputs.jsonl" echo "GH_AW_SAFE_OUTPUTS_CONFIG_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" echo "GH_AW_SAFE_OUTPUTS_TOOLS_PATH=${RUNNER_TEMP}/gh-aw/safeoutputs/tools.json" } >> "$GITHUB_OUTPUT" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" + - name: Check OTLP telemetry configuration + run: bash "${RUNNER_TEMP}/gh-aw/actions/check_otlp_default_credentials.sh" - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Create gh-aw temp directory @@ -455,6 +480,11 @@ jobs: run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_gh_for_ghe.sh" env: GH_TOKEN: ${{ github.token }} + - name: Download activation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -471,16 +501,32 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/checkout_pr_branch.cjs'); + const { main } = require(path.join(actionsDir, 'checkout_pr_branch.cjs')); await main(); - name: Install GitHub Copilot CLI - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.63 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" env: GH_HOST: github.com + GH_AW_COMPILED_VERSION: v0.88.2 - name: Install AWF binary - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.7 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.12 --rootless + - name: Determine automatic lockdown mode for GitHub MCP Server + id: determine-automatic-lockdown + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + GH_AW_GITHUB_MIN_INTEGRITY: 'none' + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const determineAutomaticLockdown = require(path.join(actionsDir, 'determine_automatic_lockdown.cjs')); + await determineAutomaticLockdown(github, context, core); - name: Parse integrity filter lists id: parse-guard-vars env: @@ -488,16 +534,11 @@ jobs: GH_AW_TRUSTED_USERS_VAR: ${{ vars.GH_AW_GITHUB_TRUSTED_USERS || '' }} GH_AW_APPROVAL_LABELS_VAR: ${{ vars.GH_AW_GITHUB_APPROVAL_LABELS || '' }} run: bash "${RUNNER_TEMP}/gh-aw/actions/parse_guard_list.sh" - - name: Download activation artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: activation - path: /tmp/gh-aw - name: Restore agent config folders from base branch if: steps.checkout-pr.outcome == 'success' env: - GH_AW_AGENT_FOLDERS: ".agents .antigravity .claude .codex .crush .gemini .github .opencode .pi" - GH_AW_AGENT_FILES: ".crush.json AGENTS.md ANTIGRAVITY.md CLAUDE.md GEMINI.md PI.md opencode.jsonc" + GH_AW_AGENT_FOLDERS: ".agents .github" + GH_AW_AGENT_FILES: "AGENTS.md" run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_base_github_folders.sh" - name: Restore inline sub-agents from activation artifact env: @@ -509,15 +550,26 @@ jobs: GH_AW_SKILL_DIR: ".github/skills" run: bash "${RUNNER_TEMP}/gh-aw/actions/restore_inline_skills.sh" - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6 ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96 ghcr.io/github/gh-aw-mcpg:v0.3.27@sha256:fe984bddde4ec05d756d9043edb0a32912e6b7b72f6a121b1082f29221421cc7 ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036 - - name: Generate Safe Outputs Config + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699 + - name: Prepare Safe Outputs Directories run: | mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_723da9f6aa534eb7_EOF' - {"add_comment":{"hide_older_comments":true,"max":1},"add_labels":{"allowed":["Auto-Triage: Waiting for Author"],"max":1},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"remove_labels":{"allowed":["Auto-Triage: Waiting for Author"],"max":1},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_723da9f6aa534eb7_EOF + - name: Generate Safe Outputs Config + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_FILE_ROOT: "${{ runner.temp }}/gh-aw" + GH_AW_FILE_CONFIG: "{\"files\":[{\"path\":\"safeoutputs/config.json\",\"content_env\":\"GH_AW_SAFE_OUTPUTS_CONFIG\"}]}" + GH_AW_SAFE_OUTPUTS_CONFIG: "{\"add_comment\":{\"hide_older_comments\":true,\"max\":1},\"add_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"remove_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"report_incomplete\":{}}" + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'create_files.cjs')); + await main(); - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -541,9 +593,18 @@ jobs: "sanitize": true, "maxLength": 65000 }, + "comment_id": { + "optionalPositiveInteger": true + }, "item_number": { "issueOrPRNumber": true }, + "pr": { + "issueOrPRNumber": true + }, + "pr_number": { + "issueOrPRNumber": true + }, "reply_to_id": { "type": "string", "maxLength": 256 @@ -551,6 +612,16 @@ jobs: "repo": { "type": "string", "maxLength": 256 + }, + "target": { + "type": "string", + "enum": [ + "status" + ] + }, + "temporary_id": { + "type": "string", + "pattern": "^#?aw_[A-Za-z0-9_]{3,12}$" } } }, @@ -562,10 +633,7 @@ jobs: }, "labels": { "required": true, - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 + "type": "array" }, "repo": { "type": "string", @@ -638,10 +706,7 @@ jobs: }, "labels": { "required": true, - "type": "array", - "itemType": "string", - "itemSanitize": true, - "itemMaxLength": 128 + "type": "array" }, "repo": { "type": "string", @@ -669,56 +734,68 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/generate_safe_outputs_tools.cjs'); + const { main } = require(path.join(actionsDir, 'generate_safe_outputs_tools.cjs')); await main(); - name: Start MCP Gateway id: start-mcp-gateway env: + GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST: ${{ vars.GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST || 'true' }} GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }} GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }} + GH_AW_SINK_VISIBILITY: ${{ steps.determine-automatic-lockdown.outputs.visibility }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eo pipefail mkdir -p "${RUNNER_TEMP}/gh-aw/mcp-config" - + if [ -n "${GITHUB_EVENT_PATH:-}" ] && [ -r "${GITHUB_EVENT_PATH}" ]; then + GH_AW_SAFEOUTPUTS_EVENT_PATH="${RUNNER_TEMP}/gh-aw/safeoutputs/github_event.json" + cp "${GITHUB_EVENT_PATH}" "${GH_AW_SAFEOUTPUTS_EVENT_PATH}" + export GITHUB_EVENT_PATH="${GH_AW_SAFEOUTPUTS_EVENT_PATH}" + fi + # Export gateway environment variables for MCP config and gateway script export MCP_GATEWAY_PORT="8080" - export MCP_GATEWAY_DOMAIN="host.docker.internal" + export MCP_GATEWAY_DOMAIN="awmg-mcpg" export MCP_GATEWAY_HOST_DOMAIN="localhost" - MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') - echo "::add-mask::${MCP_GATEWAY_API_KEY}" - export MCP_GATEWAY_API_KEY + MCP_GATEWAY_AGENT_ID=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_AGENT_ID}" + export MCP_GATEWAY_AGENT_ID export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" + export MCP_GATEWAY_ALLOWED_MOUNT_ROOTS="${GITHUB_WORKSPACE}:rw,${RUNNER_TEMP}/gh-aw:ro,${RUNNER_TEMP}/gh-aw/safeoutputs:rw,/opt:ro,/tmp:rw,/usr/bin/gh:ro" + export GH_AW_PR_HEAD_BASE_BRANCH="${GH_AW_PR_HEAD_BASE_BRANCH:-}" + export GH_AW_PR_HEAD_BASE_SHA="${GH_AW_PR_HEAD_BASE_SHA:-}" + export GH_AW_PR_HEAD_BASE_REPO="${GH_AW_PR_HEAD_BASE_REPO:-}" + export GH_AW_PR_HEAD_BASE_PR_NUMBER="${GH_AW_PR_HEAD_BASE_PR_NUMBER:-}" + export GH_AW_PR_HEAD_BASE_REF="${GH_AW_PR_HEAD_BASE_REF:-}" + export GH_AW_PR_HEAD_REPO="${GH_AW_PR_HEAD_REPO:-}" export DEBUG="*" - + export GH_AW_ENGINE="copilot" MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') - case "${DOCKER_HOST:-}" in - unix://* ) DOCKER_SOCK_PATH="${DOCKER_HOST#unix://}" ;; - /* ) DOCKER_SOCK_PATH="$DOCKER_HOST" ;; - * ) DOCKER_SOCK_PATH=/var/run/docker.sock ;; - esac - DOCKER_SOCK_GID=$(stat -c '%g' "$DOCKER_SOCK_PATH" 2>/dev/null || echo '0') - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host --name awmg-mcpg --add-host host.docker.internal:127.0.0.1 --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.3.27' - + source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_AGENT_ID -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_PR_HEAD_BASE_BRANCH -e GH_AW_PR_HEAD_BASE_SHA -e GH_AW_PR_HEAD_BASE_REPO -e GH_AW_PR_HEAD_BASE_PR_NUMBER -e GH_AW_PR_HEAD_BASE_REF -e GH_AW_PR_HEAD_REPO -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e RUNNER_TOOL_CACHE -e MCP_GATEWAY_ALLOWED_MOUNT_ROOTS -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.15' + mkdir -p "$HOME/.copilot" GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_4b806ec8e0233008_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_137b94c134aafdc9_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { "type": "stdio", - "container": "ghcr.io/github/github-mcp-server:v1.4.0", + "container": "ghcr.io/github/github-mcp-server:v1.11.0", "env": { - "GITHUB_HOST": "\${GITHUB_SERVER_URL}", - "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_FEATURES": "fields_param", + "GITHUB_HOST": "${GITHUB_SERVER_URL}", + "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_MCP_SERVER_TOKEN}", "GITHUB_READ_ONLY": "1", "GITHUB_TOOLSETS": "context,repos,issues,pull_requests" }, @@ -749,7 +826,17 @@ jobs: "GH_AW_SAFE_OUTPUTS": "\${GH_AW_SAFE_OUTPUTS}", "GH_AW_SAFE_OUTPUTS_CONFIG_PATH": "\${GH_AW_SAFE_OUTPUTS_CONFIG_PATH}", "GH_AW_SAFE_OUTPUTS_TOOLS_PATH": "\${GH_AW_SAFE_OUTPUTS_TOOLS_PATH}", + "GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST": "\${GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST}", + "GH_AW_PR_HEAD_BASE_BRANCH": "\${GH_AW_PR_HEAD_BASE_BRANCH}", + "GH_AW_PR_HEAD_BASE_SHA": "\${GH_AW_PR_HEAD_BASE_SHA}", + "GH_AW_PR_HEAD_BASE_REPO": "\${GH_AW_PR_HEAD_BASE_REPO}", + "GH_AW_PR_HEAD_BASE_PR_NUMBER": "\${GH_AW_PR_HEAD_BASE_PR_NUMBER}", + "GH_AW_PR_HEAD_BASE_REF": "\${GH_AW_PR_HEAD_BASE_REF}", + "GH_AW_PR_HEAD_REPO": "\${GH_AW_PR_HEAD_REPO}", + "GITHUB_EVENT_NAME": "\${GITHUB_EVENT_NAME}", + "GITHUB_EVENT_PATH": "\${GITHUB_EVENT_PATH}", "GITHUB_REPOSITORY": "\${GITHUB_REPOSITORY}", + "GITHUB_SHA": "\${GITHUB_SHA}", "GITHUB_TOKEN": "\${GITHUB_TOKEN}", "GITHUB_WORKSPACE": "\${GITHUB_WORKSPACE}", "RUNNER_TEMP": "\${RUNNER_TEMP}" @@ -758,7 +845,8 @@ jobs: "write-sink": { "accept": [ "*" - ] + ], + "sink-visibility": "${GH_AW_SINK_VISIBILITY}" } } } @@ -766,24 +854,32 @@ jobs: "gateway": { "port": $MCP_GATEWAY_PORT, "domain": "${MCP_GATEWAY_DOMAIN}", - "apiKey": "${MCP_GATEWAY_API_KEY}", - "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + "agentId": "${MCP_GATEWAY_AGENT_ID}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}", + "startupTimeout": 120, + "opentelemetry": { + "endpoint": "${OTEL_EXPORTER_OTLP_ENDPOINT}", + "traceId": "${GITHUB_AW_OTEL_TRACE_ID}", + "spanId": "${GITHUB_AW_OTEL_PARENT_SPAN_ID}" + } } } - GH_AW_MCP_CONFIG_4b806ec8e0233008_EOF + GH_AW_MCP_CONFIG_137b94c134aafdc9_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true env: - MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_AGENT_ID: ${{ steps.start-mcp-gateway.outputs.gateway-agent-id }} MCP_GATEWAY_DOMAIN: ${{ steps.start-mcp-gateway.outputs.gateway-domain }} MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io); - const { main } = require('${{ runner.temp }}/gh-aw/actions/mount_mcp_as_cli.cjs'); + const { main } = require(path.join(actionsDir, 'mount_mcp_as_cli.cjs')); await main(); - name: Clean credentials continue-on-error: true @@ -795,44 +891,64 @@ jobs: - name: Execute GitHub Copilot CLI id: agentic_execution # Copilot CLI tool arguments (sorted): - timeout-minutes: 20 + # --allow-tool github + # --allow-tool safeoutputs + # --allow-tool shell(cat) + # --allow-tool shell(date) + # --allow-tool shell(echo) + # --allow-tool shell(find) + # --allow-tool shell(github:*) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(ls) + # --allow-tool shell(printf) + # --allow-tool shell(pwd) + # --allow-tool shell(safeoutputs:*) + # --allow-tool shell(sort) + # --allow-tool shell(tail) + # --allow-tool shell(uniq) + # --allow-tool shell(wc) + # --allow-tool shell(yq) + # --allow-tool write + timeout-minutes: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} run: | set -o pipefail printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt - trap 'rm -f "$HOME/.copilot/settings.json"' EXIT + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"; if [ "$gh_aw_exit_code" -ne 0 ]; then echo "::error::Agent execution exited with code $gh_aw_exit_code"; fi' EXIT mkdir -p "$HOME/.copilot" printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" export XDG_CONFIG_HOME="$HOME" export GH_AW_MCP_CONFIG="$HOME/.copilot/mcp-config.json" + GH_AW_COPILOT_SRC="$(command -v copilot 2>/dev/null || true)" + if [ -z "$GH_AW_COPILOT_SRC" ] || [ ! -x "$GH_AW_COPILOT_SRC" ]; then + echo "GitHub Copilot CLI executable not found on PATH after installation" >&2 + exit 127 + fi + GH_AW_COPILOT_BIN="${RUNNER_TEMP}/gh-aw/bin/copilot" + mkdir -p "${RUNNER_TEMP}/gh-aw/bin" + if [ "$GH_AW_COPILOT_SRC" != "$GH_AW_COPILOT_BIN" ]; then + cp "$GH_AW_COPILOT_SRC" "$GH_AW_COPILOT_BIN" + fi + chmod 755 "$GH_AW_COPILOT_BIN" + touch /tmp/gh-aw/agent-step-summary.md GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) export GH_AW_NODE_BIN export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" (umask 177 && touch /tmp/gh-aw/agent-stdio.log) GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-1000}" - printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.7/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"github.com\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"ppa.launchpad.net\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"telemetry.enterprise.githubcopilot.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.5\",\"gpt-5.4\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.7,squid=sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96,agent=sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c,api-proxy=sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6,cli-proxy=sha256:4757f198a3fa20f88bdbe70be7ae1a05f127d9c0a9e96a5d6460ef40c08fc83d\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + if [[ ! "$GH_AW_MAX_AI_CREDITS" =~ ^[0-9]+$ ]]; then + GH_AW_MAX_AI_CREDITS="1000" + fi + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.28.12/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"ppa.launchpad.net\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\",\"www.googleapis.com\"],\"isolation\":true,\"topologyAttach\":[\"awmg-mcpg\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-3.7-flash\":[\"copilot/gemini-3.7*flash*\",\"google/gemini-3.7*flash*\",\"gemini/gemini-3.7*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.28.12,squid=sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f,agent=sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202,api-proxy=sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32,cli-proxy=sha256:5250629d48eaedfedf2e948785228e8da29eec2a83cbab58ea0751c14a7b021d\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" GH_AW_DOCKER_HOST="" if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then GH_AW_DOCKER_HOST="${DOCKER_HOST}" fi - GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then - GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" - python3 - <<'PY' - import json,os,subprocess as sp - from pathlib import Path - try: - p=Path(os.environ["RUNNER_TEMP"])/"gh-aw"/"awf-config.json" - c=json.loads(p.read_text()) - c["chroot"]={"binariesSourcePath":"/tmp/gh-aw","identity":{"user":sp.check_output(["id","-un"],text=True).strip(),"uid":int(sp.check_output(["id","-u"],text=True)),"gid":int(sp.check_output(["id","-g"],text=True)),"home":"/tmp/gh-aw/home"}} - out=json.dumps(c,separators=(",",":"),ensure_ascii=False)+"\n" - p.write_text(out) - Path("/tmp/gh-aw/awf-config.json").write_text(out) - except Exception as e: - raise SystemExit(f"chroot config patch failed: {e}") from e - PY + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" fi GH_AW_TOOL_CACHE_MOUNT="" GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" @@ -841,22 +957,28 @@ jobs: GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" fi fi - # shellcheck disable=SC1003,SC2086 - sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_API_KEY --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ - -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + # shellcheck disable=SC1003,SC2016,SC2086 + GH_AW_AWF_ENGINE_NAME=copilot \ + GH_AW_AWF_HARNESS_MARKER='[copilot-harness]' \ + GH_AW_AWF_LOG_FILE=/tmp/gh-aw/agent-stdio.log \ + GH_AW_AWF_ATTEMPT_LOG_NAME=copilot \ + bash "${RUNNER_TEMP}/gh-aw/actions/run_awf_with_startup_retries.sh" -- \ + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --exclude-env GITHUB_MCP_SERVER_TOKEN --exclude-env MCP_GATEWAY_AGENT_ID --mount /tmp/gh-aw:/tmp/gh-aw:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/mcp-cli/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs" "${RUNNER_TEMP}/gh-aw/bin/copilot" --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-tool github --allow-tool safeoutputs --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(date)'\'' --allow-tool '\''shell(echo)'\'' --allow-tool '\''shell(find)'\'' --allow-tool '\''shell(github:*)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(printf)'\'' --allow-tool '\''shell(pwd)'\'' --allow-tool '\''shell(safeoutputs:*)'\'' --allow-tool '\''shell(sort)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(uniq)'\'' --allow-tool '\''shell(wc)'\'' --allow-tool '\''shell(yq)'\'' --allow-tool write --allow-all-paths --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + COPILOT_MODEL: auto + GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_TIMEOUT_MINUTES: 20 - GH_AW_VERSION: v0.80.9 + GH_AW_TIMEOUT_MINUTES: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} + GH_AW_VERSION: v0.88.2 GITHUB_API_URL: ${{ github.api_url }} GITHUB_AW: true GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows @@ -876,7 +998,18 @@ jobs: if: always() id: detect-agent-errors continue-on-error: true - run: node "${RUNNER_TEMP}/gh-aw/actions/detect_agent_errors.cjs" + env: + GH_AW_AGENTIC_EXECUTION_OUTCOME: ${{ steps.agentic_execution.outcome }} + GH_AW_ENGINE_STEP_TIMEOUT_MINUTES: ${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'detect_agent_errors.cjs')); + await main(); - name: Configure Git credentials env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -892,7 +1025,7 @@ jobs: continue-on-error: true env: MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} - MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + MCP_GATEWAY_AGENT_ID: ${{ steps.start-mcp-gateway.outputs.gateway-agent-id }} GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} run: | bash "${RUNNER_TEMP}/gh-aw/actions/stop_mcp_gateway.sh" "$GATEWAY_PID" @@ -901,9 +1034,11 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); + const { main } = require(path.join(actionsDir, 'redact_secrets.cjs')); await main(); env: GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' @@ -927,25 +1062,30 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/collect_ndjson_output.cjs'); + const { main } = require(path.join(actionsDir, 'collect_ndjson_output.cjs')); await main(); - name: Parse agent logs for step summary if: always() uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_copilot_log.cjs'); + const { main } = require(path.join(actionsDir, 'parse_copilot_log.cjs')); await main(); - name: Parse MCP Gateway logs for step summary if: always() @@ -953,34 +1093,29 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + const { main } = require(path.join(actionsDir, 'parse_mcp_gateway_log.cjs')); await main(); - name: Print firewall logs if: always() continue-on-error: true env: AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs - run: | - # Fix permissions on firewall logs/audit dirs so they can be uploaded as artifacts - # AWF runs with sudo, creating files owned by root - sudo chmod -R a+rX /tmp/gh-aw/sandbox/firewall 2>/dev/null || true - # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) - if command -v awf &> /dev/null; then - awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" - else - echo 'AWF binary not installed, skipping firewall log summary' - fi + run: bash "${RUNNER_TEMP}/gh-aw/actions/print_firewall_logs.sh" --rootless - name: Parse token usage for step summary if: always() continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + const { main } = require(path.join(actionsDir, 'parse_token_usage.cjs')); await main(); - name: Print AWF reflect summary if: always() @@ -988,16 +1123,41 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/awf_reflect_summary.cjs'); + const { main } = require(path.join(actionsDir, 'awf_reflect_summary.cjs')); await main(); + - name: Generate observability summary + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'generate_observability_summary.cjs')); + await main(core); - name: Write agent output placeholder if missing if: always() run: | if [ ! -f /tmp/gh-aw/agent_output.json ]; then echo '{"items":[]}' > /tmp/gh-aw/agent_output.json fi + # Small dedicated copy of the agent output so safe-output processing + # survives a failed or timed-out upload of the larger agent artifact + - name: Upload agent output fallback artifact + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent-output-fallback + path: | + /tmp/gh-aw/agent_output.json + /tmp/gh-aw/safeoutputs.jsonl + if-no-files-found: ignore - name: Upload agent artifacts if: always() continue-on-error: true @@ -1016,6 +1176,8 @@ jobs: /tmp/gh-aw/pre-agent-audit.txt /tmp/gh-aw/agent/ /tmp/gh-aw/github_rate_limits.jsonl + /tmp/gh-aw/otel.jsonl + /tmp/gh-aw/otlp-export-errors.jsonl /tmp/gh-aw/safeoutputs.jsonl /tmp/gh-aw/agent_output.json /tmp/gh-aw/aw-*.patch @@ -1034,17 +1196,20 @@ jobs: - safe_outputs if: > always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || - needs.activation.outputs.stale_lock_file_failed == 'true' || needs.activation.outputs.daily_ai_credits_exceeded == 'true') + needs.activation.outputs.oauth_token_check_failed == 'true' || needs.activation.outputs.stale_lock_file_failed == 'true' || + needs.activation.outputs.daily_ai_credits_exceeded == 'true') runs-on: ubuntu-slim environment: issue-triage permissions: - contents: read + actions: read issues: write pull-requests: write concurrency: group: "gh-aw-conclusion-issue-triage" cancel-in-progress: false queue: max + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} outputs: incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} noop_message: ${{ steps.noop.outputs.noop_message }} @@ -1053,7 +1218,7 @@ jobs: steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -1062,15 +1227,16 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" - name: Download agent output artifact id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent + pattern: "{agent,agent-output-fallback}" + merge-multiple: true path: /tmp/gh-aw/ - name: Setup agent output environment variable id: setup-agent-output-env @@ -1078,32 +1244,29 @@ jobs: run: | mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi + - name: Download detection artifact + id: download-detection-artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: detection + path: /tmp/gh-aw/threat-detection/ + - name: Download Safe Outputs Items Manifest + id: download-safe-outputs-manifest + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: safe-outputs-items + merge-multiple: true + path: /tmp/gh-aw/ - name: Collect usage artifact files if: always() continue-on-error: true - run: | - mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection - echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw_info.json /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do - [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" - done - [ -f /tmp/gh-aw/aw_info.json ] && cp /tmp/gh-aw/aw_info.json /tmp/gh-aw/usage/aw_info.json || true - [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true - [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true - [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true - [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true - [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true - [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true - [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true - [ -f /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true - [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true - [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true - [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl - [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl - mkdir -p /tmp/gh-aw/usage/activity - node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs - find /tmp/gh-aw/usage -type f -print | sort + run: bash "${RUNNER_TEMP}/gh-aw/actions/collect_usage_artifact_files.sh" - name: Upload usage artifact if: always() continue-on-error: true @@ -1113,8 +1276,12 @@ jobs: path: | /tmp/gh-aw/usage/aw_info.json /tmp/gh-aw/usage/aw-info.jsonl + /tmp/gh-aw/usage/agent_usage.json /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/evals.jsonl + /tmp/gh-aw/usage/graders/grader_manifest.json + /tmp/gh-aw/usage/graders/grader_results.json /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl @@ -1124,7 +1291,7 @@ jobs: id: restore-daily-aic-cache-conclusion if: always() continue-on-error: true - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: key: agentic-workflow-usage-issuetriage-${{ github.run_id }} restore-keys: agentic-workflow-usage-issuetriage- @@ -1137,15 +1304,17 @@ jobs: with: github-token: ${{ github.token }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context); - const { main } = require('${{ runner.temp }}/gh-aw/actions/write_daily_aic_usage_cache.cjs'); + const { main } = require(path.join(actionsDir, 'write_daily_aic_usage_cache.cjs')); await main(); - name: Save daily AIC usage cache id: save-daily-aic-cache if: always() continue-on-error: true - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: key: agentic-workflow-usage-issuetriage-${{ github.run_id }} path: /tmp/gh-aw/agentic-workflow-usage-cache.jsonl @@ -1177,9 +1346,11 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_noop_message.cjs'); + const { main } = require(path.join(actionsDir, 'handle_noop_message.cjs')); await main(); - name: Log detection run id: detection_runs @@ -1194,9 +1365,11 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_detection_runs.cjs'); + const { main } = require(path.join(actionsDir, 'handle_detection_runs.cjs')); await main(); - name: Record missing tool id: missing_tool @@ -1209,9 +1382,11 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/missing_tool.cjs'); + const { main } = require(path.join(actionsDir, 'missing_tool.cjs')); await main(); - name: Record incomplete id: report_incomplete @@ -1224,9 +1399,11 @@ jobs: with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/report_incomplete_handler.cjs'); + const { main } = require(path.join(actionsDir, 'report_incomplete_handler.cjs')); await main(); - name: Handle agent failure id: handle_agent_failure @@ -1239,7 +1416,7 @@ jobs: GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} GH_AW_WORKFLOW_ID: "issue-triage" - GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "168" + GH_AW_ACTION_FAILURE_ISSUE_EXPIRES_HOURS: "0" GH_AW_ENGINE_ID: "copilot" GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens || '' }} @@ -1252,8 +1429,14 @@ jobs: GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} GH_AW_AGENTIC_ENGINE_TIMEOUT: ${{ needs.agent.outputs.agentic_engine_timeout }} GH_AW_MODEL_NOT_SUPPORTED_ERROR: ${{ needs.agent.outputs.model_not_supported_error }} + GH_AW_HTTP_400_RESPONSE_ERROR: ${{ needs.agent.outputs.http_400_response_error }} + GH_AW_MAX_CACHE_MISSES_EXCEEDED: ${{ needs.agent.outputs.max_cache_misses_exceeded }} + GH_AW_MISSING_MODEL_PRICING_ERROR: ${{ needs.agent.outputs.missing_model_pricing_error }} + GH_AW_MISSING_MODEL_PRICING_MODEL_NAME: ${{ needs.agent.outputs.missing_model_pricing_model_name }} + GH_AW_SHELL_EXPANSION_GUARD_REJECTED: ${{ needs.agent.outputs.shell_expansion_guard_rejected }} GH_AW_ENGINE_API_HOSTS: "api.enterprise.githubcopilot.com,api.githubcopilot.com,api.business.githubcopilot.com,api.individual.githubcopilot.com" GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} + GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }} GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} GH_AW_DAILY_AI_CREDITS_EXCEEDED: ${{ needs.activation.outputs.daily_ai_credits_exceeded }} GH_AW_DAILY_AI_CREDITS_TOTAL_EFFECTIVE_TOKENS: ${{ needs.activation.outputs.daily_ai_credits_total_effective_tokens }} @@ -1262,25 +1445,48 @@ jobs: GH_AW_FAILURE_REPORT_AS_ISSUE: "true" GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" - GH_AW_TIMEOUT_MINUTES: "20" + GH_AW_TIMEOUT_MINUTES: "${{ fromJSON(vars.GH_AW_DEFAULT_TIMEOUT_MINUTES || '20') }}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'handle_agent_failure.cjs')); + await main(); + - name: Report failed jobs + id: report_failed_jobs + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/issue-triage.md" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_REPORT_FAILED_JOBS: "true" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/handle_agent_failure.cjs'); + const { main } = require(path.join(actionsDir, 'report_failed_jobs.cjs')); await main(); detection: needs: - activation - agent - if: > - always() && needs.agent.result != 'skipped' && (needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true') + if: always() && needs.agent.result != 'skipped' runs-on: ubuntu-latest environment: issue-triage permissions: contents: read + timeout-minutes: 10 + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} outputs: aic: ${{ steps.parse_detection_token_usage.outputs.aic }} detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} @@ -1289,7 +1495,7 @@ jobs: steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -1298,15 +1504,22 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download activation artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: activation + path: /tmp/gh-aw - name: Download agent output artifact id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent + pattern: "{agent,agent-output-fallback}" + merge-multiple: true path: /tmp/gh-aw/ - name: Setup agent output environment variable id: setup-agent-output-env @@ -1314,10 +1527,12 @@ jobs: run: | mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi - name: Checkout repository for patch context if: needs.agent.outputs.has_patch == 'true' - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # --- Threat Detection --- @@ -1326,7 +1541,7 @@ jobs: rm -rf /tmp/gh-aw/sandbox/firewall/logs rm -rf /tmp/gh-aw/sandbox/firewall/audit - name: Download container images - run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.7@sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c ghcr.io/github/gh-aw-firewall/api-proxy:0.27.7@sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6 ghcr.io/github/gh-aw-firewall/squid:0.27.7@sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96 + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202 ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32 ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f - name: Check if detection needed id: detection_guard if: always() @@ -1350,21 +1565,7 @@ jobs: - name: Prepare threat detection files if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | - mkdir -p /tmp/gh-aw/threat-detection/aw-prompts - rm -f /tmp/gh-aw/agent_usage.json - cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true - if [ ! -s /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt ]; then - echo "::warning::ERR_VALIDATION: Missing or empty detection context prompt at /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt. Ensure the agent artifact includes /tmp/gh-aw/aw-prompts/prompt.txt. Detection will continue with fallback workflow context." - fi - cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true - for f in /tmp/gh-aw/aw-*.patch; do - [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true - done - for f in /tmp/gh-aw/aw-*.bundle; do - [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true - done - echo "Prepared threat detection files:" - ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + bash "${RUNNER_TEMP}/gh-aw/actions/prepare_threat_detection_files.sh" - name: Setup threat detection if: always() && steps.detection_guard.outputs.run_detection == 'true' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -1372,83 +1573,52 @@ jobs: WORKFLOW_NAME: "SqlClient Issue Auto-Triage" WORKFLOW_DESCRIPTION: "No description provided" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + GH_AW_DETECTION_SKIP_PROMPT_SUMMARY: "true" with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/setup_threat_detection.cjs'); + const { main } = require(path.join(actionsDir, 'setup_threat_detection.cjs')); await main(); - name: Ensure threat-detection directory and log if: always() && steps.detection_guard.outputs.run_detection == 'true' run: | mkdir -p /tmp/gh-aw/threat-detection touch /tmp/gh-aw/threat-detection/detection.log - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '24' - package-manager-cache: false + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.28.12 --rootless - name: Install GitHub Copilot CLI - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.63 + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" env: GH_HOST: github.com - - name: Install AWF binary - run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.7 - - name: Execute GitHub Copilot CLI + GH_AW_COMPILED_VERSION: v0.88.2 + - name: Install threat-detect binary if: always() && steps.detection_guard.outputs.run_detection == 'true' continue-on-error: true - id: detection_agentic_execution - # Copilot CLI tool arguments (sorted): - timeout-minutes: 20 run: | - set -o pipefail - printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt - trap 'rm -f "$HOME/.copilot/settings.json"' EXIT - mkdir -p "$HOME/.copilot" - printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" - export XDG_CONFIG_HOME="$HOME" - touch /tmp/gh-aw/agent-step-summary.md - GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) - export GH_AW_NODE_BIN - export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" - (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) - GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" - printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.7/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5},\"container\":{\"imageTag\":\"0.27.7,squid=sha256:deb1d4e19de62d51cee0508057a596a19315c3423ada4d675cad136dc8037c96,agent=sha256:aae231e4635c8999d039c132f1602d3df850fe9b84a00aa2b5ac981179b5661c,api-proxy=sha256:009caf2e3d88fa77b64e9a03a95a228fc58db0f1701c6d324b29ba5a3c7c79b6,cli-proxy=sha256:4757f198a3fa20f88bdbe70be7ae1a05f127d9c0a9e96a5d6460ef40c08fc83d\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" - cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json - export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" - GH_AW_DOCKER_HOST="" - if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then - GH_AW_DOCKER_HOST="${DOCKER_HOST}" - fi - GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" - if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then - GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="--docker-host-path-prefix /tmp/gh-aw" - _GH_AW_CHROOT_JSON=$(jq -c --arg src /tmp/gh-aw --arg user "$(id -un)" --argjson uid "$(id -u)" --argjson gid "$(id -g)" --arg home /tmp/gh-aw/home '.chroot={"binariesSourcePath":$src,"identity":{"user":$user,"uid":$uid,"gid":$gid,"home":$home}}' "${RUNNER_TEMP}/gh-aw/awf-config.json") || { echo "chroot config patch failed" >&2; exit 1; } - printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json" - printf '%s\n' "$_GH_AW_CHROOT_JSON" > "/tmp/gh-aw/awf-config.json" - fi - GH_AW_TOOL_CACHE_MOUNT="" - GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" - if [ -d "$GH_AW_TOOL_CACHE" ]; then - if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then - GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" - fi - fi - # shellcheck disable=SC1003,SC2086 - sudo -E awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} ${GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --audit-dir /tmp/gh-aw/sandbox/firewall/audit --enable-host-access --allow-host-ports 80,443,8080 --skip-pull \ - -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + bash "${RUNNER_TEMP}/gh-aw/actions/install_threat_detect_binary.sh" v0.5.1 + - name: Execute threat detection with AWF + id: detection_agentic_execution + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + timeout-minutes: 10 env: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + COPILOT_MODEL: auto + GH_AW_HARNESS_MAX_RETRIES: 0 + GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt - GH_AW_TIMEOUT_MINUTES: 20 - GH_AW_VERSION: v0.80.9 + GH_AW_TIMEOUT_MINUTES: 10 + GH_AW_VERSION: v0.88.2 GITHUB_API_URL: ${{ github.api_url }} GITHUB_AW: true GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows @@ -1463,58 +1633,105 @@ jobs: GIT_COMMITTER_NAME: github-actions[bot] RUNNER_TEMP: ${{ runner.temp }} TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} - - name: Parse threat detection token usage for step summary - id: parse_detection_token_usage - if: always() + WORKFLOW_NAME: "SqlClient Issue Auto-Triage" + WORKFLOW_DESCRIPTION: "No description provided" + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + GH_AW_COPILOT_SRC="$(command -v copilot 2>/dev/null || true)" + if [ -z "$GH_AW_COPILOT_SRC" ] || [ ! -x "$GH_AW_COPILOT_SRC" ]; then + echo "GitHub Copilot CLI executable not found on PATH after installation" >&2 + exit 127 + fi + GH_AW_COPILOT_BIN="${RUNNER_TEMP}/gh-aw/bin/copilot" + mkdir -p "${RUNNER_TEMP}/gh-aw/bin" + if [ "$GH_AW_COPILOT_SRC" != "$GH_AW_COPILOT_BIN" ]; then + cp "$GH_AW_COPILOT_SRC" "$GH_AW_COPILOT_BIN" + fi + chmod 755 "$GH_AW_COPILOT_BIN" + + (umask 177 && touch /tmp/gh-aw/threat-detection/detection.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + if [[ ! "$GH_AW_MAX_AI_CREDITS" =~ ^[0-9]+$ ]]; then + GH_AW_MAX_AI_CREDITS="400" + fi + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.28.12/awf-config.schema.json\",\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"detection\":[\"small\"],\"evals\":[\"small\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-3.7-flash\":[\"copilot/gemini-3.7*flash*\",\"google/gemini-3.7*flash*\",\"gemini/gemini-3.7*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"grok\":[\"copilot/*grok*\",\"openai/*grok*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.28.12,squid=sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f,agent=sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202,api-proxy=sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32,cli-proxy=sha256:5250629d48eaedfedf2e948785228e8da29eec2a83cbab58ea0751c14a7b021d\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + _GH_AW_CHROOT_JSON=$(jq -c --arg src "${RUNNER_TEMP}/gh-aw" --arg user "$(id -un)" --argjson uid "$(id -u)" --argjson gid "$(id -g)" --arg home "${RUNNER_TEMP}/gh-aw/home" '.chroot={"binariesSourcePath":$src,"identity":{"user":$user,"uid":$uid,"gid":$gid,"home":$home}}' "${RUNNER_TEMP}/gh-aw/awf-config.json") || { echo "chroot config patch failed" >&2; exit 1; } + printf '%s\n' "$_GH_AW_CHROOT_JSON" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ACTIONS_ID_TOKEN_REQUEST_TOKEN --exclude-env ACTIONS_ID_TOKEN_REQUEST_URL --exclude-env COPILOT_GITHUB_TOKEN --mount /tmp/gh-aw:/tmp/gh-aw:rw --mount /tmp/gh-aw/threat-detection:/tmp/gh-aw/threat-detection:rw --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; export PATH="${RUNNER_TEMP}/gh-aw/bin:$PATH" && : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && threat-detect --engine copilot --output /tmp/gh-aw/threat-detection/detection_result.json /tmp/gh-aw/threat-detection' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + - name: Render detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage with: script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_token_usage.cjs'); + const { main } = require(path.join(actionsDir, 'render_detection_log.cjs')); await main(); - - name: Upload threat detection log + - name: Copy detection firewall logs + if: always() && steps.detection_guard.outputs.run_detection == 'true' + continue-on-error: true + run: | + mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall + if [ -d /tmp/gh-aw/sandbox/firewall/logs ]; then mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall/logs && cp -r /tmp/gh-aw/sandbox/firewall/logs/. /tmp/gh-aw/threat-detection/sandbox/firewall/logs/; fi + if [ -d /tmp/gh-aw/sandbox/firewall/audit ]; then mkdir -p /tmp/gh-aw/threat-detection/sandbox/firewall/audit && cp -r /tmp/gh-aw/sandbox/firewall/audit/. /tmp/gh-aw/threat-detection/sandbox/firewall/audit/; fi + - name: Upload threat detection artifact if: always() && steps.detection_guard.outputs.run_detection == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: detection - path: /tmp/gh-aw/threat-detection/detection.log + path: | + /tmp/gh-aw/threat-detection/detection_result.json + /tmp/gh-aw/threat-detection/sandbox/firewall/logs/ + /tmp/gh-aw/threat-detection/sandbox/firewall/audit/ if-no-files-found: ignore - - name: Parse and conclude threat detection - id: detection_conclusion + - name: Parse threat detection token usage for step summary + id: parse_detection_token_usage if: always() continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_TOKEN_USAGE_SUMMARY_TITLE: Threat Detection Token Usage + with: + script: | + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require(path.join(actionsDir, 'parse_token_usage.cjs')); + await main(); + - name: Conclude threat detection + id: detection_conclusion + if: always() + continue-on-error: true env: RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} DETECTION_AGENTIC_EXECUTION_OUTCOME: ${{ steps.detection_agentic_execution.outcome }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" - with: - script: | - try { - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); - setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_threat_detection_results.cjs'); - await main(); - } catch (loadErr) { - const continueOnError = process.env.GH_AW_DETECTION_CONTINUE_ON_ERROR !== 'false'; - const detectionExecutionFailed = process.env.DETECTION_AGENTIC_EXECUTION_OUTCOME === 'failure'; - const msg = 'ERR_SYSTEM: \u274C Unexpected error loading threat detection module: ' + (loadErr && loadErr.message ? loadErr.message : String(loadErr)); - core.error(msg); - core.setOutput('reason', 'parse_error'); - if (continueOnError && !detectionExecutionFailed) { - core.warning('\u26A0\uFE0F ' + msg); - core.setOutput('conclusion', 'warning'); - core.setOutput('success', 'false'); - } else { - core.setOutput('conclusion', 'failure'); - core.setOutput('success', 'false'); - core.setFailed(msg); - } - } + run: | + bash "${RUNNER_TEMP}/gh-aw/actions/conclude_threat_detection.sh" /tmp/gh-aw/threat-detection/detection_result.json safe_outputs: needs: @@ -1525,7 +1742,6 @@ jobs: runs-on: ubuntu-slim environment: issue-triage permissions: - contents: read issues: write pull-requests: write timeout-minutes: 45 @@ -1538,8 +1754,8 @@ jobs: GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" - GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} - GH_AW_ENGINE_VERSION: "1.0.63" + GH_AW_ENGINE_MODEL: "auto" + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} GH_AW_WORKFLOW_ID: "issue-triage" GH_AW_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" @@ -1551,12 +1767,20 @@ jobs: comment_url: ${{ steps.process_safe_outputs.outputs.comment_url }} create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_items_applied: ${{ steps.process_safe_outputs.outputs.items_applied }} + process_safe_outputs_items_cancelled: ${{ steps.process_safe_outputs.outputs.items_cancelled }} + process_safe_outputs_items_deferred: ${{ steps.process_safe_outputs.outputs.items_deferred }} + process_safe_outputs_items_failed: ${{ steps.process_safe_outputs.outputs.items_failed }} + process_safe_outputs_items_skipped: ${{ steps.process_safe_outputs.outputs.items_skipped }} + process_safe_outputs_items_succeeded: ${{ steps.process_safe_outputs.outputs.items_succeeded }} + process_safe_outputs_items_warnings: ${{ steps.process_safe_outputs.outputs.items_warnings }} process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_status: ${{ steps.process_safe_outputs.outputs.status }} process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts id: setup - uses: github/gh-aw-actions/setup@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: destination: ${{ runner.temp }}/gh-aw/actions job-name: ${{ github.job }} @@ -1565,15 +1789,18 @@ jobs: env: GH_AW_SETUP_WORKFLOW_NAME: "SqlClient Issue Auto-Triage" GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/issue-triage.lock.yml@${{ github.ref }} - GH_AW_INFO_VERSION: "1.0.63" - GH_AW_INFO_AWF_VERSION: "v0.27.7" + GH_AW_INFO_VERSION: "1.0.80" + GH_AW_INFO_AWF_VERSION: "v0.28.12" GH_AW_INFO_ENGINE_ID: "copilot" + - name: Mask OTLP telemetry headers + run: bash "${RUNNER_TEMP}/gh-aw/actions/mask_otlp_headers.sh" - name: Download agent output artifact id: download-agent-output continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: agent + pattern: "{agent,agent-output-fallback}" + merge-multiple: true path: /tmp/gh-aw/ - name: Setup agent output environment variable id: setup-agent-output-env @@ -1581,7 +1808,9 @@ jobs: run: | mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print - echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + if [ -f "/tmp/gh-aw/agent_output.json" ]; then + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + fi - name: Configure GH_HOST for enterprise compatibility id: ghes-host-config shell: bash @@ -1597,16 +1826,18 @@ jobs: env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} - GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" + GH_AW_ALLOWED_DOMAINS: "api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,s.symcb.com,s.symcd.com,security.ubuntu.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"hide_older_comments\":true,\"max\":1},\"add_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"remove_labels\":{\"allowed\":[\"Auto-Triage: Waiting for Author\"],\"max\":1},\"report_incomplete\":{}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | - const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + const path = require('path'); + const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); + const { setupGlobals } = require(path.join(actionsDir, 'setup_globals.cjs')); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/safe_output_handler_manager.cjs'); + const { main } = require(path.join(actionsDir, 'process_safe_outputs.cjs')); await main(); - name: Upload Safe Outputs Items if: always() @@ -1616,5 +1847,5 @@ jobs: path: | /tmp/gh-aw/safe-output-items.jsonl /tmp/gh-aw/temporary-id-map.json + /tmp/gh-aw/safe-output-errors.json if-no-files-found: ignore - diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md index ef157d7f62..100d8baf54 100644 --- a/.github/workflows/issue-triage.md +++ b/.github/workflows/issue-triage.md @@ -41,6 +41,7 @@ if: | )) engine: copilot +model: auto environment: issue-triage @@ -50,6 +51,7 @@ permissions: pull-requests: read tools: + bash: [cat, find, grep] github: min-integrity: none diff --git a/.github/workflows/recheck-milestones.yml b/.github/workflows/recheck-milestones.yml new file mode 100644 index 0000000000..679b76f144 --- /dev/null +++ b/.github/workflows/recheck-milestones.yml @@ -0,0 +1,65 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# +# +# Recheck Milestones +# +# Reconciles open pull requests when a release branch is cut. +# +# check-milestone.yml decides where a milestone belongs by asking whether +# release/. exists and which milestone series is active on the +# default branch. Creating a release branch can invalidate X.Y.* pull requests +# and make the next series eligible, but emits no pull request activity, so +# already open pull requests would otherwise keep their previous verdicts. +# +# This lives in its own workflow so that check-milestone.yml stays purely pull +# request scoped, and so the elevated 'actions: write' permission needed to +# re-run checks is isolated from the PR gate. +# +# A re-run replays the original run's commit, so a pull request whose last +# milestone check predates a change to the check itself replays the older +# version and keeps its stale result. See the LIMITATION section in +# .github/scripts/recheck-milestones-for-release-branch.sh. +# +# See .github/scripts/recheck-milestones-for-release-branch.sh for the details. +# +################################################################################# + +name: Recheck Milestones + +# 'create' has no branch filter, so every branch creation starts a run of this +# workflow. The guard below skips the job for anything but release/*, which is +# why this is kept out of check-milestone.yml. +on: [create] + +jobs: + recheck-open-prs: + name: Re-check open PRs after a release branch is cut + if: github.event.ref_type == 'branch' && startsWith(github.event.ref, 'release/') + runs-on: ubuntu-latest + permissions: + # 'actions: write' is needed to re-run the affected milestone checks. + actions: write + contents: read + pull-requests: read + steps: + - name: Checkout scripts + uses: actions/checkout@v6 + with: + # A 'create' run defaults to the new branch; pin the default branch so + # the script is read from a known-good copy. + ref: ${{ github.event.repository.default_branch }} + # Only the scripts directory is needed; skip full history. + sparse-checkout: .github/scripts + sparse-checkout-cone-mode: false + + - name: Re-check affected pull requests + env: + # Pass the ref via env to avoid script injection from branch names. + RELEASE_BRANCH: ${{ github.event.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + WORKFLOW_FILE: check-milestone.yml + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash "${GITHUB_WORKSPACE}/.github/scripts/recheck-milestones-for-release-branch.sh" diff --git a/.github/workflows/verify-aw-lock.yml b/.github/workflows/verify-aw-lock.yml index 1381792980..eb3cc2e608 100644 --- a/.github/workflows/verify-aw-lock.yml +++ b/.github/workflows/verify-aw-lock.yml @@ -13,12 +13,12 @@ jobs: verify: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install gh-aw extension - uses: github/gh-aw-actions/setup-cli@8c7d04ebf1ece56cd381446125da3e0f6896294a # v0.80.9 + uses: github/gh-aw-actions/setup-cli@9271a1804551c0dc4fb0085a97979950aa2f8489 # v0.88.2 with: - version: v0.80.9 + version: v0.88.2 - name: Recompile agentic workflows run: gh aw compile diff --git a/BUILDGUIDE.md b/BUILDGUIDE.md index 72daa88ea8..55aaa62a4e 100644 --- a/BUILDGUIDE.md +++ b/BUILDGUIDE.md @@ -282,6 +282,37 @@ Package Microsoft.Data.SqlClient.Extensions.Azure without building it beforehand dotnet build -t:PackAzure -p:PackBuild=false ``` +### Release Source Link Symbols + +To reproduce release symbol generation locally, set `BuildForRelease` in the environment before +packing. `build.proj` launches child `dotnet` processes, so passing only +`-p:BuildForRelease=true` to the orchestrator does not enable it in those processes. + +```powershell +$env:BuildForRelease = 'true' +dotnet build build.proj -t:PackSqlClient -p:Configuration=Release +Remove-Item Env:\BuildForRelease +``` + +The `.nupkg` and matching `.snupkg` are written to +`artifacts/Microsoft.Data.SqlClient/Project-Release/`. Inspect them together with +[PackageValidator](tools/PackageValidator/README.md) or NuGet Package Explorer. + +To fail validation on missing source coverage or non-portable source paths: + +```powershell +dotnet run --project tools\PackageValidator\src\PackageValidator.csproj -- ` + artifacts\Microsoft.Data.SqlClient\Project-Release ` + --fail-on missing-source-link --fail-on untracked-source --fail-on non-deterministic-source-path +``` + +Tracked source paths in the PDBs must match the Source Link document map; generated sources must +be embedded. Keep the repository root configured in `RepositoryInfo.targets` as the source root: +adding a separate `src/` root without source-control metadata can remap tracked files to `/_1/` +while Source Link only maps `/_/`. NuGet Package Explorer reports this as +**Contains untracked sources (obj)** for both Source Link and Deterministic, even when all `obj` +sources are embedded and the compiler's deterministic flag is enabled. + ## Versioning Versioning can be accomplished by using a mix of different parameters to the `build.proj` targets: diff --git a/CHANGELOG.md b/CHANGELOG.md index dd60021553..505801db29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,113 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) > **Note:** Releases are sorted in reverse chronological order (newest first). +## [Stable Release 7.1.0] - 2026-09-17 + +General availability of Microsoft.Data.SqlClient 7.1. The sections below list the changes since [7.1.0-preview3](release-notes/7.1/7.1.0-preview3.md). See the [7.1.0 release notes](release-notes/7.1/7.1.0.md) for the cumulative list of changes since the 7.0.3 stable release. + +### Added + +- Added a `RegisteredApplication` enum and a matching `SqlConnection.RegisteredApplication` property that let a library or tool identify itself to SQL Server through version 2 of the TDS USERAGENT feature extension. The payload also carries a new driver-owned 64-bit *Driver Properties* flag field; bit 0 reports whether connection pool V2 is enabled for the process. Application identity is client-supplied telemetry and must never be used for authorization or any other security decision. The value must be set before `Open`/`OpenAsync` and is not part of the connection pool key. + ([#3201](https://github.com/dotnet/SqlClient/issues/3201), [#4632](https://github.com/dotnet/SqlClient/pull/4632)) + +### Changed + +- `SqlConnectionStringBuilder.TransparentNetworkIPResolution` is now marked `[Obsolete]`, directing callers to `MultiSubnetFailover`. There is no runtime behavior change: connection string defaults are untouched and no new AppContext switches were introduced. The only visible effect is a `CS0618` build warning for code that references the property. + ([#4494](https://github.com/dotnet/SqlClient/issues/4494), [#4576](https://github.com/dotnet/SqlClient/pull/4576)) + +- Unified the exception message raised when conflicting token-based and SSPI authentication properties are set on the same `SqlConnection`, and documented the complete set of properties that conflict with `AccessToken`. + ([#4629](https://github.com/dotnet/SqlClient/pull/4629)) + +- Documentation corrections for `SqlDataRecord`, `SqlMetaData`, and the LCID 1033 locale name. + ([#1805](https://github.com/dotnet/SqlClient/issues/1805), + [#4440](https://github.com/dotnet/SqlClient/pull/4440), + [#4646](https://github.com/dotnet/SqlClient/pull/4646)) + +- Updated `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` to v7.1.0 (was v7.1.0-preview3.26226.3). + ([#4698](https://github.com/dotnet/SqlClient/pull/4698)) + +### Fixed + +- Fixed a pooled connection being returned to the pool in a broken state after a `TransactionScope` rollback — for example, when distributed transaction promotion fails on .NET 8+ where implicit distributed transactions are disabled by default. Connection reset now preserves the transaction when the pooled connection is either a delegated transaction root or enlisted in a transaction. + ([#4001](https://github.com/dotnet/SqlClient/issues/4001), [#4557](https://github.com/dotnet/SqlClient/pull/4557)) + +- Fixed `GetSchema("DataTypes")` never reporting the SQL Server 2025 `json` type against Azure SQL. The decision now uses the `json` support flag negotiated through the TDS `FEATUREEXTACK` token instead of a server version string comparison. + ([#4592](https://github.com/dotnet/SqlClient/issues/4592), [#4682](https://github.com/dotnet/SqlClient/pull/4682)) + +- Fixed a malformed UNC pipe path being composed for IPv6 literal server names over Named Pipes in managed SNI, which could trigger an access violation inside LSASS on Windows and force a reboot. IPv6 literals are now transcribed to their `.ipv6-literal.net` form. (net8.0/net9.0 only) + ([#4523](https://github.com/dotnet/SqlClient/issues/4523), [#4558](https://github.com/dotnet/SqlClient/pull/4558)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now installed only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547)) + +- Fixed open/close throughput regressions in the opt-in connection pool V2 (`Switch.Microsoft.Data.SqlClient.UseConnectionPoolV2`). The default pool is unaffected. + ([#4543](https://github.com/dotnet/SqlClient/pull/4543)) + +### Companion packages + +- Released `Microsoft.Data.SqlClient.Extensions.Azure` 7.1.0 with an internal Entra ID authority parsing clarification and no behavior change. See [release notes](release-notes/Extensions/Azure/7.1/7.1.0.md). +- Released version-aligned `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, and `Microsoft.Data.SqlClient.Internal.Logging` 7.1.0 with no functional or API changes since preview3. See the [Azure Key Vault provider](release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md), [Abstractions](release-notes/Extensions/Abstractions/7.1/7.1.0.md), and [Logging](release-notes/Internal/Logging/7.1/7.1.0.md) release notes. + +## [Stable Release 7.0.3] - 2026-09-10 + + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4599](https://github.com/dotnet/SqlClient/pull/4599)) + +### Fixed + +- Fixed a `SqlBulkCopy` regression in environments where the application login cannot read `sys.all_columns`. Bulk copy now falls back to the earlier column-discovery behavior when that permission is unavailable. Support for hidden columns and SQL Graph column aliases still requires access to the metadata view. + ([#4370](https://github.com/dotnet/SqlClient/issues/4370), [#4306](https://github.com/dotnet/SqlClient/pull/4306), [#4402](https://github.com/dotnet/SqlClient/pull/4402)) + +- Fixed a memory-allocation regression in connection and command operations caused by formatting diagnostic strings even when tracing was disabled. Also corrected trace messages that reported an incorrect object ID or could throw `FormatException` when traced values contained braces. + ([#4528](https://github.com/dotnet/SqlClient/pull/4528), [#4533](https://github.com/dotnet/SqlClient/pull/4533)) + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4583](https://github.com/dotnet/SqlClient/pull/4583)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4553](https://github.com/dotnet/SqlClient/pull/4553)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561)) + +- Fixed authentication state handling so clearing `SqlConnection.AccessToken`, `AccessTokenCallback`, or `SspiContextProvider` preserves the other authentication values in the connection pool key. Cloning a connection or updating its credential also preserves its `SspiContextProvider`. Combining a non-null `SspiContextProvider` with `AccessToken` or `AccessTokenCallback` now throws `InvalidOperationException` instead of silently discarding authentication state; applications must use one authentication mechanism at a time. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561), [#4644](https://github.com/dotnet/SqlClient/pull/4644)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4663](https://github.com/dotnet/SqlClient/pull/4663)) + +### Companion packages + +- Released `Microsoft.Data.SqlClient.Extensions.Azure` 7.0.3 with the Entra ID authority parsing fix for Dataverse/Dynamics 365 connections. See [release notes](release-notes/Extensions/Azure/7.0/7.0.3.md). +- Released version-aligned `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider`, `Microsoft.Data.SqlClient.Extensions.Abstractions`, and `Microsoft.Data.SqlClient.Internal.Logging` 7.0.3 with no functional or API changes. See the [Azure Key Vault provider](release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md), [Abstractions](release-notes/Extensions/Abstractions/7.0/7.0.3.md), and [Logging](release-notes/Internal/Logging/7.0/7.0.3.md) release notes. + +## [Stable Release 6.1.7] - 2026-09-10 + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4598](https://github.com/dotnet/SqlClient/pull/4598)) + +### Fixed + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4584](https://github.com/dotnet/SqlClient/pull/4584)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4552](https://github.com/dotnet/SqlClient/pull/4552)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed token authentication state handling so clearing `SqlConnection.AccessToken` preserves an existing `AccessTokenCallback` in the connection pool key, and clearing `AccessTokenCallback` preserves an existing `AccessToken`. Callback-based authentication now also follows the same prelogin server-certificate validation rules as an explicitly supplied access token. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4664](https://github.com/dotnet/SqlClient/pull/4664)) + ## [Preview Release 7.1.0-preview3] - 2026-08-26 This update brings the following changes since the [7.1.0-preview2](release-notes/7.1/7.1.0-preview2.md) release. @@ -168,7 +275,7 @@ See the [full release notes](release-notes/7.1/7.1.0-preview2.md) for detailed d - `SqlVector` now serializes and deserializes multibyte values as little-endian explicitly. ([#3861](https://github.com/dotnet/SqlClient/pull/3861)) -- Updated the bundled .NET 10 SDK to `10.0.300`. +- Updated the .NET 10 SDK used to build the repository to `10.0.300`. ([#4287](https://github.com/dotnet/SqlClient/pull/4287)) - Re-shipped `Microsoft.Data.SqlClient.Extensions.Azure` as `7.1.0-preview2`, adding Windows Account Manager (WAM) broker support for Entra ID authentication on Windows. See [release notes](release-notes/Extensions/Azure/7.1/7.1.0-preview2.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 13ee1ef83b..df139c5482 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -132,7 +132,7 @@ Security issues and bugs should be reported privately, via email, to the Microso ## Submitting Pull Requests - **New features from community PRs must be driven by creating a GitHub issue first.** Discuss the proposal in the issue before starting implementation. This helps avoid wasted effort and ensures alignment with project goals. -- **Community contributions must not derail the project roadmap.** We prioritize features and fixes according to our published milestones. PRs that conflict with or distract from active roadmap items may be deferred. +- **Community contributions must align with project priorities.** We prioritize features and fixes according to our [published milestones](https://github.com/dotnet/SqlClient/milestones). PRs that conflict with or distract from active work may be deferred. - **Our maintainers reserve the right to reject PRs** that do not meet the required criteria to qualify for review. This includes PRs that: - Lack a corresponding approved issue (i.e., an issue that has been reviewed, acknowledged, and agreed upon by maintainers — typically indicated by the **`PM Approved`** field being set to **`Approved`** in the GitHub Project board) - Introduce breaking changes without prior discussion @@ -171,7 +171,7 @@ Some important caveats: ## Contribution Standards -Project maintainers will merge changes that improve the product significantly and broadly and that align with the [Microsoft.Data.SqlClient roadmap](roadmap.md). +Project maintainers will merge changes that improve the product significantly and broadly and that align with the [Microsoft.Data.SqlClient roadmap](https://github.com/dotnet/SqlClient/wiki/Roadmap). ### Requirements diff --git a/Directory.Packages.props b/Directory.Packages.props index 03ad924f6d..ad71a41be4 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -52,7 +52,7 @@ --> - 7.1.0-preview3.26226.3 + 7.1.0 [$(SniVersion), $([MSBuild]::Add($(SniVersion.Split('.')[0]), 1)).0.0) @@ -108,6 +108,24 @@ + + + + + + + diff --git a/NuGet.analysis.config b/NuGet.analysis.config new file mode 100644 index 0000000000..1ba52a93d3 --- /dev/null +++ b/NuGet.analysis.config @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/NuGet.config b/NuGet.config index a16ff70302..5bdc9722d5 100644 --- a/NuGet.config +++ b/NuGet.config @@ -5,19 +5,43 @@ + + + - + + + + + + + + + + + - - + + + + + + + + + + diff --git a/README.md b/README.md index 07374c727b..bf77802f15 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,9 @@ When targeting .NET on Windows, a package reference to [Microsoft.Data.SqlClient | Coding Style | [coding-style.md](/policy/coding-style.md) | | Contributing | [CONTRIBUTING.md](CONTRIBUTING.md) | | Copyright Information | [COPYRIGHT.md](COPYRIGHT.md) | -| Roadmap | [roadmap.md](roadmap.md) | +| Release Milestones | [GitHub milestones](https://github.com/dotnet/SqlClient/milestones) | | Review Process | [review-process.md](/policy/review-process.md) | +| Roadmap | [SqlClient roadmap](https://github.com/dotnet/SqlClient/wiki/Roadmap) | | Support Policy | [SUPPORT.md](SUPPORT.md) | ## Our Featured Contributors diff --git a/TESTGUIDE.md b/TESTGUIDE.md index c813db0345..206092b597 100644 --- a/TESTGUIDE.md +++ b/TESTGUIDE.md @@ -220,6 +220,7 @@ Update `config.jsonc` for your environment before running manual tests. The most "NPConnectionString": "Data Source=np:localhost;Database=Northwind;Integrated Security=true;Encrypt=false;", "EnclaveEnabled": false, "TracingEnabled": false, + "SupportsEntraIntegrated": false, "SupportsIntegratedSecurity": true } ``` @@ -247,6 +248,9 @@ dotnet build -t:TestSqlClientManual -p:TestSet=2 ## Configuration Properties +`SupportsEntraIntegrated` applies to Azure SQL Database, Azure SQL Managed Instance, and SQL Server +2022 or later configured for Microsoft Entra authentication through Azure Arc. + | Property | Description | Example or notes | |----------------------------------|---------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| | `TCPConnectionString` | Connection string for a TCP-enabled SQL Server or Azure SQL database. | `Data Source=tcp:localhost;Database=Northwind;Integrated Security=true;Encrypt=false;` | @@ -262,6 +266,7 @@ dotnet build -t:TestSqlClientManual -p:TestSet=2 | `AADServicePrincipalSecret` | Optional application secret for service-principal authentication tests. | Keep this only in local, ignored config files or secure pipeline variables. | | `AzureKeyVaultURL` | Optional Azure Key Vault URL for Always Encrypted tests. | `https://.vault.azure.net/` | | `AzureKeyVaultTenantId` | Optional Entra ID tenant ID for Azure Key Vault tests. | Tenant ID GUID. | +| `SupportsEntraIntegrated` | Whether the target supports Entra Integrated authentication for the Windows identity. | `true` or `false`; defaults to `false`. See supported targets above. | | `SupportsIntegratedSecurity` | Whether the user running tests has integrated-security access to the target SQL Server. | `true` or `false`. | | `LocalDbAppName` | Optional LocalDB instance name. Empty disables LocalDB testing. | `MSSQLLocalDB` or another local instance. | | `LocalDbSharedInstanceName` | Optional shared LocalDB instance name. | Used only when testing shared LocalDB. | diff --git a/build.proj b/build.proj index c29dec53a1..4dfef2b77b 100644 --- a/build.proj +++ b/build.proj @@ -96,6 +96,21 @@ packages — use PackageVersionSqlClient (below) to set the version for the entire family. --> + + + + -p:SqlClientNextVersion=$(SqlClientNextVersion) + + + + + -p:SqlClientFileVersion=$(FileVersionSqlClient) + + + + + + -p:SqlServerNextVersion=$(SqlServerNextVersion) + + + + + -p:SqlServerFileVersion=$(FileVersionSqlServer) + + false + + + + --no-incremental + -p:ArtifactPath="$(IsolatedBuildPath)/bin/" + + + + false + + -p:EnableAnalyzers=true + + + + false + $(EnableAnalyzersArgument) + -p:InternalAnalyzers=true + + + + $(EnableAnalyzersArgument) + -p:RestoreConfigFile="$(InternalAnalyzersNugetConfig)" + + + + $(EnableAnalyzersArgument) + -p:InternalAnalyzersVersion=$(InternalAnalyzersVersion) + + - <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlClientNextVersionArgument) <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) - <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlClientProjectPath)" -getProperty:SqlClientFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlClientNextVersionArgument) <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) @@ -370,7 +555,7 @@ - <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerPackageVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlServerNextVersionArgument) <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) @@ -380,7 +565,7 @@ - <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) + <_Cmd>"$(DotnetPath)dotnet" build "$(SqlServerProjectPath)" -getProperty:SqlServerFileVersion $(BuildNumberArgument) $(BuildSuffixArgument) $(SqlServerNextVersionArgument) <_Cmd>$([System.Text.RegularExpressions.Regex]::Replace($(_Cmd), "\s+", " ")) @@ -459,6 +644,8 @@ "$(DotnetPath)dotnet" build "$(SqlClientNotSupportedProjectPath)" -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) -p:GenApiPath="@(GenApiArtifactPath->'%(FullPath)')" $(SigningKeyPathArgument) @@ -466,6 +653,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -489,12 +677,15 @@ "$(DotnetPath)dotnet" build $(SqlClientRefProjectPath) -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -518,12 +709,15 @@ "$(DotnetPath)dotnet" build $(SqlClientProjectPath) -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -567,6 +761,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -719,12 +914,15 @@ "$(DotnetPath)dotnet" build "$(AkvProviderProjectPath)" -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -752,6 +950,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -824,12 +1023,15 @@ "$(DotnetPath)dotnet" build "$(AbstractionsProjectPath)" -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -856,6 +1058,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -927,12 +1130,15 @@ "$(DotnetPath)dotnet" build "$(AzureProjectPath)" -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -959,6 +1165,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $(ReferenceTypeArgument) @@ -1027,12 +1234,15 @@ "$(DotnetPath)dotnet" build $(LoggingProjectPath) -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) @@ -1056,6 +1266,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlClientArgument) + $(FileVersionSqlClientArgument) $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) @@ -1091,12 +1302,15 @@ "$(DotnetPath)dotnet" build $(SqlServerProjectPath) -p:Configuration=$(Configuration) + $(IsolatedBuildArgument) + $(EnableAnalyzersArgument) $(SigningKeyPathArgument) $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlServerArgument) + $(FileVersionSqlServerArgument) $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) @@ -1120,6 +1334,7 @@ $(BuildNumberArgument) $(BuildSuffixArgument) $(PackageVersionSqlServerArgument) + $(FileVersionSqlServerArgument) $([System.Text.RegularExpressions.Regex]::Replace($(DotnetCommand), "\s+", " ")) diff --git a/doc/design-notes/4001-delegated-transaction-reset.md b/doc/design-notes/4001-delegated-transaction-reset.md new file mode 100644 index 0000000000..79d24fb04c --- /dev/null +++ b/doc/design-notes/4001-delegated-transaction-reset.md @@ -0,0 +1,322 @@ +# Root cause analysis: #4001 — pooled connection broken after `TransactionScope` rollback + +| | | +|---|---| +| **Issue** | [#4001](https://github.com/dotnet/SqlClient/issues/4001) | +| **Regressed by** | [#3019](https://github.com/dotnet/SqlClient/pull/3019) (`0322d44c7`), shipped in 6.1.0 | +| **Affected** | 6.1.0 → 6.1.6, `main` | +| **Last good** | 6.0.5 | +| **Code** | `SqlConnectionInternal.ResetConnection()` | + +This note records why the bug happens, why the previous fix caused it, why the new +condition cannot reintroduce the issue that fix was addressing, and — importantly — +what the existing test suite does and does not actually verify. + +--- + +## 1. Background: two ways a connection can be "in" a transaction + +This distinction is the crux of the entire bug. + +When a connection participates in a `TransactionScope`, it ends up in one of **two +distinct** states. They overlap, but neither implies the other. + +### Delegated root — "I *own* this transaction" + +Only one connection is involved, so `System.Transactions` delegates the transaction +down to SQL Server rather than paying for a distributed coordinator. The transaction +lives **on** the connection. + +- `IsTransactionRoot` → `true` +- `EnlistedTransaction` → set at first, then **cleared** later (see below) + +`IsTransactionRoot` is not a stored flag. It is derived: + +```csharp +internal bool IsTransactionRoot => DelegatedTransaction?.IsActive == true; +``` + +Enlistment sets `EnlistedTransaction` unconditionally, so a *freshly* delegated root +has both. But once the transaction is no longer `Active`, +`DbConnectionInternal.DetachCurrentTransactionIfEnded` clears `EnlistedTransaction`: + +```csharp +transactionIsDead = enlistedTransaction.TransactionInformation.Status != TransactionStatus.Active; +if (transactionIsDead) { DetachTransaction(enlistedTransaction, true); } +``` + +The delegated transaction, meanwhile, can still report `IsActive == true`. That +transient **half-state** — root, but no `EnlistedTransaction` — is precisely the state +issue #4001 reproduces in. + +### Enlisted participant — "I *joined* someone else's transaction" + +Multiple resources are involved, so a coordinator (MSDTC) owns the transaction and +each connection enlists in it. + +- `IsTransactionRoot` → `false` +- `EnlistedTransaction` → set + +### The trap + +Neither field subsumes the other: a delegated root can have a `null` +`EnlistedTransaction`, and an enlisted participant is never a root. Any check that +tests only one of these fields silently misses the other case — and does so with no +exception at the point of the mistake. + +--- + +## 2. Where the damage occurs + +When a connection is closed it returns to the pool and is **reset** — wiped clean for +the next consumer. If a transaction is still in flight, that reset must *preserve* it. +The entire decision is one boolean: + +```csharp +_parser.PrepareResetConnection(preserveTransaction); +``` + +Pass `false` while a transaction is genuinely live, and the TDS reset destroys the +server-side transaction **while `System.Transactions` still believes it exists**. + +The failure then surfaces later, some distance from the cause: + +1. `TransactionScope` disposes and rolls back. +2. `SqlDelegatedTransaction.Rollback` asks the server to roll back a transaction the + server no longer has. +3. The rollback fails, and SqlClient calls `DoomThisConnection()`. +4. The physical connection is now permanently marked broken. +5. With a small pool (the report used `MaxPoolSize=1`) that same doomed connection is + immediately handed back out. +6. The next caller gets: + +> `InvalidOperationException: The requested operation cannot be completed because the connection has been broken.` + +The exception names the connection, not the reset that ruined it. That distance +between cause and symptom is what makes this class of bug hard to trace. + +--- + +## 3. What PR #3019 actually changed + +The relevant diff from `0322d44c7`: + +```diff +- _parser.PrepareResetConnection(IsTransactionRoot && !IsNonPoolableTransactionRoot); ++ _parser.PrepareResetConnection(EnlistedTransaction is not null && Pool is not null); +``` + +The old helper was: + +```csharp +internal protected override bool IsNonPoolableTransactionRoot + => IsTransactionRoot && (!Is2008OrNewer || Pool == null); +``` + +Substituting, the pre-#3019 condition was: + +```csharp +IsTransactionRoot && Is2008OrNewer && Pool != null +``` + +The `Is2008OrNewer` term is **deliberately not carried forward**, for two reasons. + +First, it is outside the support matrix. The term is `false` for exactly one server +version — SQL Server 2005 — and the driver's supported floor is SQL Server 2012. + +Second, and more importantly, **it was never safe on its own.** `IsNonPoolableTransactionRoot` +had two jobs, not one. Besides suppressing the preserve bit, it also drove pool routing: +`DbConnectionPool` sent any connection it flagged into *stasis* rather than back into the +pool. Being parked is what made a plain reset harmless — the connection was never handed +to another caller. #3019 deleted the property entirely, and today's pools route on +`EnlistedTransaction` alone. A delegated root with a `null` `EnlistedTransaction` now goes +straight back to the **general** pool. + +Reinstating only the suppression half would therefore reset a live delegated transaction +*and* hand the connection out again — #4001 exactly, just narrowed to SQL Server 2005. +Half of a retired safety mechanism is worse than none of it. + +So the two conditions were: + +| | Question it asked | Covered | Missed | +|---|---|---|---| +| **Pre-#3019** | "Am I the *owner*?" | delegated root | enlisted → **#2970** | +| **#3019** | "Am I *enlisted*?" | enlisted | delegated root → **#4001** | + +**#3019 swapped one case for the other rather than covering both.** It genuinely fixed +#2970, and it traded it for #4001. Both conditions were half-right; neither was wrong +about the case it did cover. + +This reframes the fix: the goal is not to undo #3019, it is to finish it. + +--- + +## 4. The fix + +The predicate is extracted into a helper so it can be tested directly: + +```csharp +internal static bool ShouldPreserveTransactionOnReset( + bool isPooled, + bool isTransactionRoot, + bool hasEnlistedTransaction) +{ + if (!isPooled) + { + return false; + } + + return isTransactionRoot || hasEnlistedTransaction; +} +``` + +This is `OLD || NEW`: each arm answers one of the two questions from section 1, so the +predicate is `true` wherever either predecessor was. + +--- + +## 5. Why this cannot reintroduce #2970 + +This is the question that matters most, and it is answerable by inspection rather than +by testing. Every reachable state, for a pooled connection: + +| `IsTransactionRoot` | `EnlistedTransaction` | Pre-#3019 | #3019 | **This fix** | +|:---:|:---:|:---:|:---:|:---:| +| `false` | `null` | `false` | `false` | `false` | +| **`true`** | **`null`** | ✅ `true` | ❌ `false` ← **#4001** | ✅ **`true`** | +| **`false`** | **set** | ❌ `false` ← **#2970** | ✅ `true` | ✅ **`true`** | +| `true` | set | `true` | `true` | `true` | + +Read the **#2970 row**. That is the row PR #3019 was created to fix, and this fix still +evaluates `true` there. It is untouched. + +Reintroducing #2970 would require that cell to flip to `false`, and `A || B` cannot +evaluate `false` while `B` is `true`. The guarantee is structural, not empirical. + +Because each arm reproduces its original predecessor exactly, the condition returns +`true` wherever either predecessor did, and never returns `false` where one of them +returned `true`. + +--- + +## 6. How the root cause was established + +The cause was **proven at runtime, not inferred**. The driver was temporarily +instrumented at the reset site and at `DoomThisConnection()`. The captured state at the +critical reset: + +``` +[RESET] obj=4 preserve=False root=False deleg=null enlisted=null pool=set +[RESET] obj=7 preserve=False root=False deleg=null enlisted=null pool=set +[RESET] obj=7 preserve=False root=True deleg=active=True enlisted=null pool=set <-- old: true, new: false +[DOOM] obj=7 + at Microsoft.Data.SqlClient.SqlDelegatedTransaction.Rollback(...) + at System.Transactions.Transaction.Rollback() + at System.Transactions.TransactionScope.InternalDispose() + at System.Transactions.TransactionScope.Dispose() +[FAIL] Bug reproduced +``` + +The third reset is the bug caught in the act: `root=True`, `deleg.IsActive=True`, +`enlisted=null`, and `preserve=False`. A live delegated transaction being discarded. + +This mattered, because **the initial hypothesis was wrong.** The first theory was that +#3019 had made the condition *too broad*, and a narrowing fix was written on that +basis. It did not work. The instrumentation showed the opposite — #3019 had *narrowed* +the condition, not widened it — and the fix was rewritten accordingly. Without runtime +evidence this would have been fixed in the wrong direction. + +All instrumentation was removed before commit. + +--- + +## 7. What the existing tests actually verify + +This section is deliberately blunt, because the intuitive answer is wrong. + +### Bisection + +Against the reporter's reproduction (NHibernate 5.5.2, `MaxPoolSize=1`, +`TransactionScope` with a failed DTC promotion): + +**6.0.5 ✅ · 6.1.0 ❌ · 6.1.1 ❌ · 6.1.4 ❌ · `main` ❌** + +This places the regression in the 6.1.0 window, consistent with #3019. + +### Both pool implementations, both directions + +| | without fix | with fix | +|---|---|---| +| `WaitHandleDbConnectionPool` (default) | ❌ reproduces | ✅ passes | +| `ChannelDbConnectionPool` (`UseConnectionPoolV2`) | ❌ reproduces | ✅ passes | + +The **left-hand column is the load-bearing one.** It was produced by stashing the fix +and rebuilding. Without it, a pass on the V2 pool could simply mean V2 never reaches +this code path, which would prove nothing. + +(V2 in released 6.1.4 throws `NotImplementedException`, so only `main` was testable.) + +### Mutation testing of the manual suite + +`--filter "FullyQualifiedName~TransactionTest"` reports **9/9 passing** with the fix. +That number is easy to over-read, so the suite was mutation-tested: the condition was +replaced with each known-buggy variant and the suite re-run. + +| Condition compiled in | Bug it contains | Suite result | +|---|---|---| +| Pre-#3019 (`IsTransactionRoot && Pool is not null`) | **#2970** | **9/9 passed** | +| #3019 (`EnlistedTransaction is not null && Pool is not null`) | **#4001** | **9/9 passed** | +| This fix (union) | none | 9/9 passed | + +**The suite passes on all three.** It does not detect either bug, and therefore does +not guard this line at all in this environment. + +`Test_EnlistedTransactionPreservedWhilePooled` — the test added by #3019 specifically +to cover #2970 — is tagged `[Trait("Category", "flaky")]` and passes against code that +carries the #2970 bug. + +The practical conclusion: **the 9/9 result is evidence of no collateral damage, not +evidence that the fix works.** + +### The regression test that was added + +An end-to-end reproduction was attempted extensively and abandoned. The `#4001` state +requires a narrow simultaneity — the transaction's status already non-`Active` (so +`DetachCurrentTransactionIfEnded` has cleared `EnlistedTransaction`) while +`DelegatedTransaction.IsActive` is still `true` — and which of two teardown paths in +`SqlDelegatedTransaction` wins is a race: + +- `TransactionEnded` sets `_active = false` and *immediately* calls + `DoomThisConnection()`. If this path runs first, the delegate is already inactive + before any reset, so the state is never observed. +- `Rollback`, driven from `TransactionScope.Dispose`, sets `_active = false` *after* the + reset. This is the ordering the reporter hit. + +Roughly twenty harness variants — varying pool size, pool implementation, promotion +success, explicit rollback, and parking the delegate in the transacted pool ahead of the +enlistment — consistently drove the first path. A test built on that race would be +flaky, which is the same defect `Test_EnlistedTransactionPreservedWhilePooled` already +demonstrates. + +Instead the predicate was extracted into +`SqlConnectionInternal.ShouldPreserveTransactionOnReset` and pinned directly by +`SqlConnectionInternalResetTransactionTests`, following the existing precedent of +`ResolveLoginTimeout` / `SqlConnectionInternalTimeoutTests`. The tests were themselves +mutation-tested: + +| Condition compiled into the helper | Bug it contains | New tests | +|---|---|---| +| `hasEnlistedTransaction` (#3019) | **#4001** | ❌ 1 failed | +| `isTransactionRoot` (pre-#3019) | **#2970** | ❌ 1 failed | +| The shipped condition | none | ✅ 8 passed | + +This satisfies "fails before the change, passes after" for **both** regressions, and +does so deterministically and without a server. + +--- + +## 8. Related + +- **#2970** — the issue #3019 was fixing. Fully preserved by this change (section 5). +- **#2285** — reports the same exception with no reproduction. Plausibly the same root + cause, though unconfirmed. diff --git a/doc/samples/SqlDataRecord.cs b/doc/samples/SqlDataRecord.cs index f81de5b850..18981dc7db 100644 --- a/doc/samples/SqlDataRecord.cs +++ b/doc/samples/SqlDataRecord.cs @@ -2,10 +2,9 @@ namespace SqlDataRecordCS; using System; +using System.Collections.Generic; using System.Data; -using System.Data.Sql; -using System.Data.SqlTypes; -using Microsoft.SqlServer.Server; +using Microsoft.Data.SqlClient.Server; public sealed partial class SqlDataRecordTester { @@ -13,41 +12,40 @@ private SqlDataRecordTester() { } -[SqlProcedure] -public static void CallTestMethods() -{ - CreateNewRecord(); - CreateNewRecord1(); - -} - // +//using System; +//using System.Collections.Generic; +//using System.Data; //using Microsoft.Data.SqlClient.Server; -[SqlProcedure] -public static void CreateNewRecord() +// Stream rows to SQL Server as a table-valued parameter. +public static IEnumerable GetRecords() { - - // Variables. - SqlDataRecord record; + // Re-use a single SqlDataRecord instance rather than allocating a new one for each row. + // Each row's values are read before SqlCommand advances to the next one. + SqlDataRecord record; - // Create a new record with the column metadata. The constructor is - // able to accept a variable number of parameters. - record = new SqlDataRecord(new SqlMetaData[] { new SqlMetaData("Column1", SqlDbType.NVarChar, 12), + // Create a new record with the column metadata. The constructor is + // able to accept a variable number of parameters. + record = new SqlDataRecord(new SqlMetaData[] { new SqlMetaData("Column1", SqlDbType.NVarChar, 12), new SqlMetaData("Column2", SqlDbType.Int), new SqlMetaData("Column3", SqlDbType.DateTime) }); - // Set the record fields. - record.SetString(0, "Hello World!"); - record.SetInt32(1, 42); - record.SetDateTime(2, DateTime.Now); + // Set the record fields. + record.SetString(0, "Hello World!"); + record.SetInt32(1, 42); + record.SetDateTime(2, DateTime.Now); - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Stream the first record to SQL Server. + yield return record; + + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } // -public static void CreateNewRecord1() +public static void CreateNewRecord() { // @@ -65,15 +63,11 @@ public static void CreateNewRecord1() // Create a new record with the column metadata. record = new SqlDataRecord(new SqlMetaData[] { column1Info, column2Info }); -// - // Set the record fields. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); -// Send the record to the calling program. -SqlContext.Pipe.Send(record); - +// } } #endif diff --git a/doc/samples/SqlMetaData.cs b/doc/samples/SqlMetaData.cs index c23cf363b7..fd6e2fff6b 100644 --- a/doc/samples/SqlMetaData.cs +++ b/doc/samples/SqlMetaData.cs @@ -2,10 +2,9 @@ namespace SqlMetaDataCS; using System; +using System.Collections.Generic; using System.Data; -using System.Data.Sql; -using System.Data.SqlTypes; -using Microsoft.SqlServer.Server; +using Microsoft.Data.SqlClient.Server; public sealed partial class SqlMetaDataTester { @@ -14,10 +13,12 @@ private SqlMetaDataTester() } // + // using System; + // using System.Collections.Generic; + // using System.Data; // using Microsoft.Data.SqlClient.Server; - [SqlProcedure] - public static void CreateNewRecord() + public static IEnumerable ReturnNewRecords() { // Variables. SqlMetaData column1Info; @@ -35,13 +36,15 @@ public static void CreateNewRecord() column2Info, column3Info }); - // Set the record fields. + // Set the fields of the first record and stream it to SQL Server. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); record.SetDateTime(2, DateTime.Now); + yield return record; - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } // diff --git a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlDataRecord.xml b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlDataRecord.xml index 7b0af6f6c9..c8b69e73c9 100644 --- a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlDataRecord.xml +++ b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlDataRecord.xml @@ -1,4 +1,4 @@ - + @@ -6,25 +6,28 @@ - This class is used together with to send result sets to the client from managed code stored-procedures. + This class describes a single row of a table-valued parameter. Construct one from an array of objects which describe the column metadata of the record, populate it with the Set<Type> methods, include it in an IEnumerable<SqlDataRecord> and assign that IEnumerable<SqlDataRecord> to . - When writing common language runtime (CLR) applications, you should re-use existing SqlDataRecord objects instead of creating new ones every time. Creating many new SqlDataRecord objects could severely deplete memory and adversely affect performance. + For best performance when sending multiple rows, re-use a single instance across an IEnumerable<SqlDataRecord> iterator: set new column values and the same instance for each row. will read each record before advancing to the next. - The following example shows how to create several objects, which describe the column metadata of a record, and creating a . The column values of the are set and the is sent to the calling program by using the class. + The following example shows how to create several objects describing the column metadata of a record, then to create a re-usable instance. The column values of the are set and the same can be used to stream multiple rows to SQL Server as a table-valued parameter. + using System; + using System.Collections.Generic; + using System.Data; using Microsoft.Data.SqlClient.Server; - - [Microsoft.Data.SqlClient.Server.SqlProcedure] - public static void CreateNewRecord() + + // Stream rows to SQL Server as a table-valued parameter. + public static IEnumerable<SqlDataRecord> CreateNewRecord() { - - // Variables. + // Re-use a single SqlDataRecord instance rather than allocating a new one for each row. + // Each row's values are read before SqlCommand advances to the next one. SqlDataRecord record; // Create a new record with the column metadata. The constructor is @@ -38,8 +41,12 @@ record.SetInt32(1, 42); record.SetDateTime(2, DateTime.Now); - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Stream the first record to SQL Server. + yield return record; + + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } @@ -75,9 +82,6 @@ // Set the record fields. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); - - // Send the record to the calling program. - SqlContext.Pipe.Send(record); diff --git a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml index cbabfd8cb0..492e85e46b 100644 --- a/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml +++ b/doc/snippets/Microsoft.Data.SqlClient.Server/SqlMetaData.xml @@ -1,19 +1,24 @@ - + - Specifies and retrieves metadata information from parameters and columns of objects. This class cannot be inherited. + Specifies the metadata information of a column used by a object. This class cannot be inherited. + + instances are typically built once and reused to construct many objects which are streamed to SQL Server as a table-valued parameter. For more information, see Table-Valued Parameters. + - The following example shows the creation of several objects, which describe the column metadata of a record, and the creation of a . The column values of the are set and the is sent to the calling program using the class. + The following example shows the creation of several objects, which describe the column metadata of a record, and the generation of a stream of records. These records can be streamed to SQL Server as a table-valued parameter by assigning the return value of the method to the property. + using System; + using System.Collections.Generic; + using System.Data; using Microsoft.Data.SqlClient.Server; - - [Microsoft.Data.SqlClient.Server.SqlProcedure] - public static void CreateNewRecord() + + public static IEnumerable<SqlDataRecord> ReturnNewRecords() { // Variables. SqlMetaData column1Info; @@ -32,13 +37,15 @@ column2Info, column3Info }); - // Set the record fields. + // Set the fields of the first record and stream it to SQL Server. record.SetString(0, "Hello World!"); record.SetInt32(1, 42); record.SetDateTime(2, DateTime.Now); + yield return record; - // Send the record to the calling program. - SqlContext.Pipe.Send(record); + // Set the fields of the second record and stream it to SQL Server. + record.SetInt32(1, 0); + yield return record; } @@ -456,7 +463,7 @@ The SQL Server type name for . - Initializes a new instance of the class with the specified column name, user-defined type (UDT), and SQLServer type. + Initializes a new instance of the class with the specified column name, user-defined type (UDT), and SQL Server type. diff --git a/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml b/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml new file mode 100644 index 0000000000..58dc683dcb --- /dev/null +++ b/doc/snippets/Microsoft.Data.SqlClient/RegisteredApplication.xml @@ -0,0 +1,147 @@ + + + + + Specifies the known application identifiers that Microsoft.Data.SqlClient reports for user agent telemetry. + + + + Production applications that meet the bar are welcome to reserve an identifier here. + + + Identifier reservations are as follows: + + + + 0x0001-0x7FFF: Microsoft-defined large-scale applications. + + + 0x8000-0xBFFF: Reserved for small-scale use. + + + 0xC000-0xFFFF: Public and developer use. + + + + An unregistered identifier may still be reported by casting a value to this type. + + + + + + No application identity is reported. This is the default. + + + 0 + + + + + The Microsoft Entity Framework Core SQL Server provider. + + + 1 + + + + + Microsoft Semantic Kernel. + + + 2 + + + + + Microsoft SQL Server Management Studio. + + + 3 + + + + + Microsoft SQL Server Management Objects. + + + 4 + + + + + Microsoft SQL Server Data-Tier Application Framework. + + + 5 + + + + + Microsoft SQL Tools Service. + + + 6 + + + + + Microsoft ASP.NET Core distributed SQL Server cache. + + + 7 + + + + + Microsoft Entity Framework 6 SQL Server provider. + + + 8 + + + + + Microsoft Azure Functions SQL extension. + + + 9 + + + + + Microsoft Orleans ADO.NET providers. + + + 10 + + + + + Microsoft Durable Task SQL Server provider. + + + 11 + + + + + The sqlpackage command-line tool. + + + 12 + + + sqlpackage is built on the Data-Tier Application Framework, but reports its own identifier so that + command-line use can be told apart from other callers of that framework. + + + + + Microsoft Data API builder. + + + 13 + + + + diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml index 05f3c774b0..d663d13a36 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnection.xml @@ -262,10 +262,11 @@ The following example creates a and a This property is mutually exclusive with the - + , + , and - properties, among others. Setting this property when + properties. Setting this property when is already set throws , because SSPI is an @@ -360,9 +361,14 @@ The following example creates a and a are the same, they will be grouped into the same connection pool. - When using a token callback function, the connection manages - refreshing the tokens returned by the callback. The application is - not responsible for knowing when tokens expire. + The driver manages token refresh and discards pooled connections with + expired or nearly expired tokens before reuse. Connections in use or + reused within the same active transaction are unaffected. + + + New physical connections invoke the callback only when a cached token + is missing or needs refreshing. Reusing a pooled connection does not + invoke the callback. This property is mutually exclusive with the @@ -1033,7 +1039,7 @@ The following table lists the valid names for keyword values within the
-or-

ServerSPN|N/A|The SPN for the data source. The default value is an empty string, which causes SqlClient to use the default, driver-generated SPN.

(Only available in v5.0+)| |Transaction Binding|Implicit Unbind|Controls connection association with an enlisted `System.Transactions` transaction.

Possible values are:

`Transaction Binding=Implicit Unbind;`

`Transaction Binding=Explicit Unbind;`

Implicit Unbind causes the connection to detach from the transaction when it ends. After detaching, additional requests on the connection are performed in autocommit mode. The `System.Transactions.Transaction.Current` property is not checked when executing requests while the transaction is active. After the transaction has ended, additional requests are performed in autocommit mode.

If the system ends the transaction (in the scope of a using block) before the last command completes, it will throw .

Explicit Unbind causes the connection to remain attached to the transaction until the connection is closed or an explicit `SqlConnection.TransactionEnlist(null)` is called. Beginning in .NET Framework 4.0, changes to Implicit Unbind make Explicit Unbind obsolete. An `InvalidOperationException` is thrown if `Transaction.Current` is not the enlisted transaction or if the enlisted transaction is not active.| -|Transparent Network IP Resolution

-or-

TransparentNetworkIPResolution|See description.|On .NET Framework, when the value of this key is set to `true`, the driver runs multiple connect rounds across the DNS-resolved IP addresses, with progressively larger per-attempt timeouts and a 500 ms minimum on the sequential-mode attempt, until a connection succeeds or the overall `Connect Timeout` is reached.

If the `MultiSubnetFailover` key is set to `true`, `TransparentNetworkIPResolution` is ignored.

If the `Failover Partner` key is set, `TransparentNetworkIPResolution` is ignored.

On .NET Framework, if `TransparentNetworkIPResolution` isn't specified in the connection string, the driver automatically disables TNIR when the data source is an Azure SQL endpoint (`.database.windows.net`, `.database.cloudapi.de`, `.database.usgovcloudapi.net`, `.database.chinacloudapi.cn`, or `.database.fabric.microsoft.com`), when the `Authentication` key is set to any Microsoft Entra ID method (`Active Directory Password`, `Active Directory Integrated`, `Active Directory Interactive`, `Active Directory Service Principal`, `Active Directory Device Code Flow`, `Active Directory Managed Identity`, `Active Directory MSI`, `Active Directory Default`, or `Active Directory Workload Identity`), or when the or property is set on the . For these automatic conditions, an explicit `TransparentNetworkIPResolution` value bypasses the automatic behavior: `True` enables TNIR, and `False` disables TNIR unconditionally. To restore the automatic behavior, remove the keyword from the connection string.

On .NET (Core, .NET 5+), `TransparentNetworkIPResolution` isn't a recognized connection-string keyword. Setting it (with any value) throws `ArgumentException` when the driver parses the connection string.

On .NET Framework, the value of this key must be `true`, `false`, `yes`, or `no`.

A value of `yes` is treated the same as a value of `true`.

A value of `no` is treated the same as a value of `false`.| +|Transparent Network IP Resolution

-or-

TransparentNetworkIPResolution|See description.|**Deprecated.** Use `Multi Subnet Failover` instead.

On .NET Framework, when the value of this key is set to `true`, the driver runs multiple connect rounds across the DNS-resolved IP addresses, with progressively larger per-attempt timeouts and a 500 ms minimum on the sequential-mode attempt, until a connection succeeds or the overall `Connect Timeout` is reached.

If the `MultiSubnetFailover` key is set to `true`, `TransparentNetworkIPResolution` is ignored.

If the `Failover Partner` key is set, `TransparentNetworkIPResolution` is ignored.

On .NET Framework, if `TransparentNetworkIPResolution` isn't specified in the connection string, the driver automatically disables TNIR when the data source is an Azure SQL endpoint (`.database.windows.net`, `.database.cloudapi.de`, `.database.usgovcloudapi.net`, `.database.chinacloudapi.cn`, or `.database.fabric.microsoft.com`), when the `Authentication` key is set to any Microsoft Entra ID method (`Active Directory Password`, `Active Directory Integrated`, `Active Directory Interactive`, `Active Directory Service Principal`, `Active Directory Device Code Flow`, `Active Directory Managed Identity`, `Active Directory MSI`, `Active Directory Default`, or `Active Directory Workload Identity`), or when the or property is set on the . For these automatic conditions, an explicit `TransparentNetworkIPResolution` value bypasses the automatic behavior: `True` enables TNIR, and `False` disables TNIR unconditionally. To restore the automatic behavior, remove the keyword from the connection string.

On .NET (Core, .NET 5+), `TransparentNetworkIPResolution` isn't a recognized connection-string keyword. Setting it (with any value) throws when the driver parses the connection string.

On .NET Framework, the value of this key must be `true`, `false`, `yes`, or `no`.

A value of `yes` is treated the same as a value of `true`.

A value of `no` is treated the same as a value of `false`.| |Trust Server Certificate

-or-

TrustServerCertificate|'false'|When set to `true`, TLS is used to encrypt the channel when bypassing walking the certificate chain to validate trust. If TrustServerCertificate is set to `true` and Encrypt is set to `false`, the channel is not encrypted. Recognized values are `true`, `false`, `yes`, and `no`. For more information, see [Connection String Syntax](https://learn.microsoft.com/sql/connect/ado-net/connection-string-syntax).| |Type System Version|N/A|A string value that indicates the type system the application expects. The functionality available to a client application is dependent on the version of SQL Server and the compatibility level of the database. Explicitly setting the type system version that the client application was written for avoids potential problems that could cause an application to break if a different version of SQL Server is used. **Note:** The type system version cannot be set for common language runtime (CLR) code executing in-process in SQL Server. For more information, see [SQL Server Common Language Runtime Integration](https://learn.microsoft.com/dotnet/framework/data/adonet/sql/sql-server-common-language-runtime-integration).

Possible values are:

`Type System Version=SQL Server 2012;`

`Type System Version=SQL Server 2008;`

`Type System Version=SQL Server 2005;`

`Type System Version=Latest;`

`Type System Version=SQL Server 2012;` specifies that the application will require version 11.0.0.0 of Microsoft.SqlServer.Types.dll. The other `Type System Version` settings will require version 10.0.0.0 of Microsoft.SqlServer.Types.dll.

`Latest` is obsolete and should not be used. `Latest` is equivalent to `Type System Version=SQL Server 2008;`.| |User ID

-or-

UID

-or-

User|N/A|The SQL Server login account. Not recommended. To maintain a high level of security, we strongly recommend that you use the `Integrated Security` or `Trusted_Connection` keywords instead. is a more secure way to specify credentials for a connection that uses SQL Server Authentication.

The user ID must be 128 characters or less.| @@ -2294,6 +2300,56 @@ The following sample tries to open a connection to an invalid database to simula
+ + + Gets or sets the middleware application identity reported to the server for this connection. + + + A value. The default is + . + + + + Set the identity before opening the connection: + + + using Microsoft.Data.SqlClient; + + using SqlConnection connection = new(connectionString) + { + RegisteredApplication = RegisteredApplication.EntityFrameworkCore + }; + connection.Open(); + + + + The connection is opening or open. The identity is reported during login, so it must be set beforehand. + + + + This API is intended for registered applications that reserve an identifier in + . An unregistered identifier may be reported by + casting a value to that type. + + + This value is telemetry. It is supplied entirely by the client, which may report any identifier in range, + so it is not an authenticated identity and must not be used for authorization or any other security + decision. + + + The identity is sent once, during login, so it must be set before the connection is opened. + + + When pooling is enabled the value is reported only while establishing a new physical connection, and it is + not part of the pool key. A connection served from the pool therefore reports the identity of whichever + connection caused that physical connection to be created, and physical connections opened in the background + to satisfy Min Pool Size report + . Applications that mix identities over one + connection string should treat this telemetry as indicative rather than exact, or disable pooling where an + exact attribution is required. + + + Gets a string that identifies the database client. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml index 1387196677..ffeb1ad240 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlConnectionStringBuilder.xml @@ -1416,6 +1416,9 @@ This property corresponds to the "ServerSPN" and "Server SPN" keys within the co A boolean value. + + This property is obsolete. Use instead. + If the Multi Subnet Failover key is set to true, Transparent Network IP Resolution is ignored. diff --git a/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml b/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml index ae7dbdd931..6559612464 100644 --- a/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml +++ b/doc/snippets/Microsoft.Data.SqlClient/SqlParameter.xml @@ -505,7 +505,7 @@ The following example creates multiple instances of diff --git a/eng/pipelines/ci/kerberos/build-and-test-steps.yml b/eng/pipelines/ci/kerberos/build-and-test-steps.yml deleted file mode 100644 index dbb3b5e1e2..0000000000 --- a/eng/pipelines/ci/kerberos/build-and-test-steps.yml +++ /dev/null @@ -1,138 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# Shared build-and-test steps used by both the Windows and Linux Kerberos jobs. -# -# Parameters: -# testFramework — The TFM to test against (e.g. net9.0, net462). -# testRunTitle — Title for the published test results (displayed in the ADO -# Tests tab). -# artifactName — Name of the published pipeline artifact that carries the -# test results and coverage files. - -parameters: - - # TFM to pass to the test targets (-p:TestFramework). - - name: testFramework - type: string - - # Title shown in the ADO Tests tab for this run. - - name: testRunTitle - type: string - - # Pipeline artifact name for test results and coverage. - - name: artifactName - type: string - -steps: - - # --------------------------------------------------------------------------- - # Build - # --------------------------------------------------------------------------- - - # Build SqlClient. - # - # The test stages build as part of their targets, but this separate step isolates build failures - # so we can fail fast before running tests. Retries are enabled intentionally (1 attempt for - # build, 2 attempts for test steps) to reduce transient infrastructure-related failures. - # - - task: DotNetCoreCLI@2 - displayName: Build SqlClient - retryCountOnTaskFailure: 1 - inputs: - command: build - projects: build.proj - arguments: >- - -t:BuildSqlClient - -p:Configuration=Release - - # --------------------------------------------------------------------------- - # Run tests in separate steps to permit focused retries. - # --------------------------------------------------------------------------- - - # Run the Unit Test suite. - - task: DotNetCoreCLI@2 - displayName: Run Unit Tests (${{ parameters.testFramework }}) - retryCountOnTaskFailure: 2 - inputs: - command: build - projects: build.proj - arguments: >- - -t:TestSqlClientUnit - -p:TestFramework=${{ parameters.testFramework }} - -p:Configuration=Release - - # Run the Functional Test suite. - - task: DotNetCoreCLI@2 - displayName: Run Functional Tests (${{ parameters.testFramework }}) - retryCountOnTaskFailure: 2 - inputs: - command: build - projects: build.proj - arguments: >- - -t:TestSqlClientFunctional - -p:TestFramework=${{ parameters.testFramework }} - -p:Configuration=Release - - # Run the Manual Test suite. - - task: DotNetCoreCLI@2 - displayName: Run Manual Tests (${{ parameters.testFramework }}) - retryCountOnTaskFailure: 2 - inputs: - command: build - projects: build.proj - arguments: >- - -t:TestSqlClientManual - -p:TestFramework=${{ parameters.testFramework }} - -p:Configuration=Release - - # --------------------------------------------------------------------------- - # Publish results & coverage - # --------------------------------------------------------------------------- - - # Publish the TRX test results to the pipeline run. - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - inputs: - testResultsFormat: VSTest - # build.proj defines TestResultsFolderPath which defaults to - # $(Build.SourcesDirectory)/test_results, so we look there for the results and coverage files. - testResultsFiles: $(Build.SourcesDirectory)/test_results/**/*.trx - mergeTestResults: true - testRunTitle: ${{ parameters.testRunTitle }} - buildConfiguration: Release - - # Azure Pipelines task conditions do not support path existence checks directly, - # so compute this once and gate later steps on the variable. - - pwsh: | - $resultsDir = "$(Build.SourcesDirectory)/test_results" - if (Test-Path -LiteralPath $resultsDir) { - Write-Host "##vso[task.setvariable variable=HasTestResultsDir]true" - } - else { - Write-Host "##vso[task.setvariable variable=HasTestResultsDir]false" - } - displayName: Detect test_results directory - condition: succeededOrFailed() - - # Give our coverage files a unique name to make it clear where they originated when we download - # the artifacts from all jobs in the merge stage. - - pwsh: | - cd $(Build.SourcesDirectory)/test_results - Get-ChildItem -Filter "*.coverage" -Recurse | - Rename-Item -NewName { "${{ parameters.testFramework }}" + $_.Name } - displayName: Rename coverage files - condition: and(succeededOrFailed(), eq(variables['HasTestResultsDir'], 'true')) - - # Publish TRX test results and coverage files as pipeline artifacts. The merge stage needs the - # coverage files from all of the jobs. - - task: PublishPipelineArtifact@1 - displayName: Publish Test Artifacts - condition: and(succeededOrFailed(), eq(variables['HasTestResultsDir'], 'true')) - inputs: - targetPath: $(Build.SourcesDirectory)/test_results - artifact: ${{ parameters.artifactName }} diff --git a/eng/pipelines/ci/kerberos/linux-init-step.yml b/eng/pipelines/ci/kerberos/linux-init-step.yml deleted file mode 100644 index c0c4603232..0000000000 --- a/eng/pipelines/ci/kerberos/linux-init-step.yml +++ /dev/null @@ -1,117 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# - -# This template joins a Linux agent to an Active Directory domain using Kerberos -# and acquires a TGT (Ticket-Granting Ticket) for the specified domain user. -# -# Prerequisites: -# - The agent must be running on Ubuntu/Debian (uses apt-get). -# - The domain controller must be reachable from the agent network. -# -# After this step completes successfully, the agent will have: -# - Kerberos packages installed (krb5-user, realmd, sssd, adcli, etc.) -# - Hostname set to FQDN within the domain -# - NTP synchronized with the domain controller -# - Machine joined to the AD domain -# - A valid Kerberos TGT for the specified user - -parameters: - - # The Active Directory domain to join (e.g. mydomain.contoso.com). - - name: kerberosDomain - type: string - - # The Organizational Unit in which to place the computer account. - - name: kerberosDomainOU - type: string - - # The domain user account to authenticate with (sAMAccountName, without @realm). - - name: kerberosDomainUser - type: string - - # The password for the domain user account. - - name: kerberosDomainPassword - type: string - -steps: - - - bash: | - set -euo pipefail - - DOMAIN="${{ parameters.kerberosDomain }}" - DOMAIN_OU="${{ parameters.kerberosDomainOU }}" - DOMAIN_USER="${{ parameters.kerberosDomainUser }}" - DOMAIN_PASSWORD="${{ parameters.kerberosDomainPassword }}" - DOMAIN_UPPER=$(echo "$DOMAIN" | tr '[:lower:]' '[:upper:]') - - echo "Domain: $DOMAIN" - echo "Realm: $DOMAIN_UPPER" - echo "User: $DOMAIN_USER" - echo "OU: $DOMAIN_OU" - - if [ -z "$DOMAIN_PASSWORD" ]; then - echo "##vso[task.logissue type=error]KerberosDomainPassword is empty" - exit 1 - fi - - # ----------------------------------------------------------------------- - # Install Kerberos and AD integration packages - # ----------------------------------------------------------------------- - echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections - - sudo apt-get -y update - sudo apt-get install -y dialog apt-utils - sudo apt-get install -y \ - krb5-user samba sssd sssd-tools libnss-sss libpam-sss \ - ntp ntpdate realmd adcli - - # ----------------------------------------------------------------------- - # Set the hostname to FQDN within the domain - # ----------------------------------------------------------------------- - CURRENT_HOSTNAME="$(hostname)" - if [ "$CURRENT_HOSTNAME" = "$DOMAIN" ] || [[ "$CURRENT_HOSTNAME" == *".$DOMAIN" ]]; then - echo "Hostname already uses domain suffix '.$DOMAIN': $CURRENT_HOSTNAME" - else - sudo hostnamectl set-hostname "$CURRENT_HOSTNAME.$DOMAIN" - fi - - # ----------------------------------------------------------------------- - # Synchronize time with the domain controller (required for Kerberos) - # ----------------------------------------------------------------------- - if ! sudo grep -Fqx "server $DOMAIN" /etc/ntp.conf; then - echo "server $DOMAIN" | sudo tee -a /etc/ntp.conf - fi - sudo systemctl stop ntp - sudo ntpdate "$DOMAIN" - sudo systemctl start ntp - - # ----------------------------------------------------------------------- - # Configure Kerberos realm - # ----------------------------------------------------------------------- - echo "[libdefaults] - default_realm = $DOMAIN_UPPER - rdns = false" | sudo tee /etc/krb5.conf - - # ----------------------------------------------------------------------- - # Discover and join the domain - # ----------------------------------------------------------------------- - sudo realm discover "$DOMAIN_UPPER" - - echo "$DOMAIN_PASSWORD" | sudo realm join --verbose "$DOMAIN_UPPER" \ - -U "$DOMAIN_USER@$DOMAIN_UPPER" \ - --computer-ou "OU=$DOMAIN_OU" - - realm list - - # ----------------------------------------------------------------------- - # Acquire a Kerberos TGT - # ----------------------------------------------------------------------- - echo "$DOMAIN_PASSWORD" | kinit "$DOMAIN_USER@$DOMAIN_UPPER" - - klist - sudo ip addr - sudo ip route - displayName: Initialize Kerberos (domain join + kinit) diff --git a/eng/pipelines/ci/kerberos/linux-setup-step.yml b/eng/pipelines/ci/kerberos/linux-setup-step.yml new file mode 100644 index 0000000000..c5de26b3ba --- /dev/null +++ b/eng/pipelines/ci/kerberos/linux-setup-step.yml @@ -0,0 +1,110 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Configures a Linux agent, joins it to the domain, acquires a Kerberos ticket, and verifies SQL +# connectivity before running integration tests. + +parameters: + + - name: kerberosDomain + type: string + + - name: kerberosDomainOU + type: string + + - name: kerberosDomainUser + type: string + + - name: kerberosDomainPassword + type: string + +steps: + + - pwsh: | + $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json + foreach ($p in $jdata) { + $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING + $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING + $p.SupportsIntegratedSecurity = $true + } + $jdata | Add-Member -NotePropertyName "KerberosDomainUser" -NotePropertyValue $env:KERBEROS_DOMAIN_USER -Force + $jdata | Add-Member -NotePropertyName "KerberosDomainPassword" -NotePropertyValue $env:KERBEROS_DOMAIN_PASSWORD -Force + $jdata | ConvertTo-Json | Set-Content "config.jsonc" + workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities + displayName: Update test config.jsonc (Kerberos) + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) + REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) + KERBEROS_DOMAIN_USER: ${{ parameters.kerberosDomainUser }} + KERBEROS_DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} + + - bash: | + set -euo pipefail + + DOMAIN="${{ parameters.kerberosDomain }}" + DOMAIN_OU="${{ parameters.kerberosDomainOU }}" + DOMAIN_USER="${{ parameters.kerberosDomainUser }}" + DOMAIN_UPPER=$(echo "$DOMAIN" | tr '[:lower:]' '[:upper:]') + + echo "Domain: $DOMAIN" + echo "Realm: $DOMAIN_UPPER" + echo "User: $DOMAIN_USER" + echo "OU: $DOMAIN_OU" + + if [ -z "${DOMAIN_PASSWORD:-}" ]; then + echo "##vso[task.logissue type=error]KerberosDomainPassword is empty" + exit 1 + fi + + echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + + sudo apt-get -y update + sudo apt-get install -y dialog apt-utils + sudo apt-get install -y \ + krb5-user samba sssd sssd-tools libnss-sss libpam-sss \ + ntp ntpdate realmd adcli + + CURRENT_HOSTNAME="$(hostname)" + if [ "$CURRENT_HOSTNAME" = "$DOMAIN" ] || [[ "$CURRENT_HOSTNAME" == *".$DOMAIN" ]]; then + echo "Hostname already uses domain suffix '.$DOMAIN': $CURRENT_HOSTNAME" + else + sudo hostnamectl set-hostname "$CURRENT_HOSTNAME.$DOMAIN" + fi + + if ! sudo grep -Fqx "server $DOMAIN" /etc/ntp.conf; then + echo "server $DOMAIN" | sudo tee -a /etc/ntp.conf + fi + sudo systemctl stop ntp + sudo ntpdate "$DOMAIN" + sudo systemctl start ntp + + echo "[libdefaults] + default_realm = $DOMAIN_UPPER + rdns = false" | sudo tee /etc/krb5.conf + + sudo realm discover "$DOMAIN_UPPER" + + echo "$DOMAIN_PASSWORD" | sudo realm join --verbose "$DOMAIN_UPPER" \ + -U "$DOMAIN_USER@$DOMAIN_UPPER" \ + --computer-ou "OU=$DOMAIN_OU" + + realm list + + echo "$DOMAIN_PASSWORD" | kinit "$DOMAIN_USER@$DOMAIN_UPPER" + + klist + sudo ip addr + sudo ip route + displayName: Initialize Kerberos (domain join + kinit) + env: + DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} + + - pwsh: | + Install-Module -Name SqlServer -Force -Confirm:$false + Import-Module SqlServer + Invoke-Sqlcmd -Query "SELECT @@VERSION, @@SERVERNAME" -ConnectionString $env:REMOTE_TCP_CONN_STRING + displayName: Verify SQL connectivity + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) \ No newline at end of file diff --git a/eng/pipelines/ci/kerberos/linux-cleanup-step.yml b/eng/pipelines/ci/kerberos/linux-teardown-step.yml similarity index 77% rename from eng/pipelines/ci/kerberos/linux-cleanup-step.yml rename to eng/pipelines/ci/kerberos/linux-teardown-step.yml index d3491bb337..df01baffc0 100644 --- a/eng/pipelines/ci/kerberos/linux-cleanup-step.yml +++ b/eng/pipelines/ci/kerberos/linux-teardown-step.yml @@ -4,9 +4,9 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This template leaves the Active Directory domain and destroys Kerberos -# credentials. It should be referenced at the end of any job that called -# linux-init-step.yml. +# This template tears down the Linux Kerberos environment by leaving the Active Directory domain +# and destroying credentials. It should be referenced at the end of any job that called +# linux-setup-step.yml. # # All steps use condition: always() so that cleanup runs even when previous # steps fail. @@ -32,14 +32,15 @@ steps: DOMAIN="${{ parameters.kerberosDomain }}" DOMAIN_USER="${{ parameters.kerberosDomainUser }}" - DOMAIN_PASSWORD="${{ parameters.kerberosDomainPassword }}" DOMAIN_UPPER=$(echo "$DOMAIN" | tr '[:lower:]' '[:upper:]') # Leave the domain - echo "$DOMAIN_PASSWORD" | sudo realm leave "$DOMAIN_UPPER" --verbose \ + echo "${DOMAIN_PASSWORD:-}" | sudo realm leave "$DOMAIN_UPPER" --verbose \ -U "$DOMAIN_USER@$DOMAIN_UPPER" || true # Destroy the TGT and credential cache kdestroy || true displayName: Clean up Kerberos (domain leave + kdestroy) condition: always() + env: + DOMAIN_PASSWORD: ${{ parameters.kerberosDomainPassword }} diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml new file mode 100644 index 0000000000..0a8efa104c --- /dev/null +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml @@ -0,0 +1,137 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Runs the Kerberos integration tests for one OS, runtime, and SNI configuration using the exact +# packages produced by the triggering sqlclient-ci-package pipeline. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: displayName + type: string + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: jobNameSuffix + type: string + + - name: operatingSystem + type: string + values: + - Linux + - Windows + + # The pool VM image to use. The pool named by poolName must provide an image with this name. + # + # NOTE: This value is evaluated at template-expansion (compile) time to select the pool image, so + # it must be a literal and must not contain any runtime expressions (e.g. $(...) macros or + # $[...] runtime expressions). + - name: poolImage + type: string + + - name: poolName + type: string + + - name: runtime + type: string + + - name: useManagedSNI + type: boolean + default: false + +jobs: + - job: kerberos_tests_job_${{ parameters.jobNameSuffix }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 90 + + workspace: + clean: all + + pool: + name: ${{ parameters.poolName }} + demands: + - ImageOverride -equals ${{ parameters.poolImage }} + + steps: + + # Align source with the commit that produced the upstream packages while retaining the + # pipeline definitions from the commit at which this run was queued. + - template: /eng/pipelines/common/steps/align-source-with-upstream-step.yml@self + + - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self + + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + parameters: + debug: ${{ parameters.debug }} + runtimes: [8.x, 9.x, 10.x] + + - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self + + - task: NuGetAuthenticate@1 + displayName: Authenticate NuGet feeds + + - ${{ if eq(parameters.operatingSystem, 'Windows') }}: + - template: /eng/pipelines/ci/kerberos/windows-setup-step.yml@self + parameters: + useManagedSNI: ${{ parameters.useManagedSNI }} + + - ${{ if eq(parameters.operatingSystem, 'Linux') }}: + - template: /eng/pipelines/ci/kerberos/linux-setup-step.yml@self + parameters: + kerberosDomain: $(KerberosDomain) + kerberosDomainOU: $(KerberosDomainOU) + kerberosDomainUser: $(KerberosDomainUser) + kerberosDomainPassword: $(KerberosDomainPassword) + + # UnitTests and FunctionalTests cover environment-independent SPN, SSPI, and connection + # string behavior in normal CI. Run only ManualTests that use the Kerberos environment. + - task: DotNetCoreCLI@2 + displayName: Run Kerberos Integration Tests + retryCountOnTaskFailure: 2 + inputs: + command: build + projects: build.proj + arguments: >- + --verbosity ${{ parameters.dotnetVerbosity }} + -t:TestSqlClientManual + -p:TestFramework=${{ parameters.runtime }} + -p:TestSet=3 + -p:TestFilters="category!=failing&category!=flaky&category!=interactive&(FullyQualifiedName~KerberosTests|FullyQualifiedName~IntegratedAuthenticationTest|FullyQualifiedName~InstanceNameTest)" + -p:ReferenceType=Package + -p:Configuration=${{ parameters.buildConfiguration }} + -p:PackageVersionSqlClient=$(sqlClientPackageVersion) + -p:PackageVersionSqlServer=$(sqlServerPackageVersion) + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFormat: VSTest + testResultsFiles: $(Build.SourcesDirectory)/test_results/**/*.trx + mergeTestResults: true + testRunTitle: ${{ parameters.displayName }} + buildConfiguration: ${{ parameters.buildConfiguration }} + + - ${{ if eq(parameters.operatingSystem, 'Linux') }}: + - template: /eng/pipelines/ci/kerberos/linux-teardown-step.yml@self + parameters: + kerberosDomain: $(KerberosDomain) + kerberosDomainUser: $(KerberosDomainUser) + kerberosDomainPassword: $(KerberosDomainPassword) \ No newline at end of file diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml index 6bdea0157f..28ce2a2ed6 100644 --- a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-pipeline.yml @@ -1,55 +1,63 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### -# ============================================================================= -# sqlclient-ci-kerberos-pipeline -# ============================================================================= -# Daily Kerberos authentication test pipeline for Microsoft.Data.SqlClient. -# Replaces the Classic "Test-SqlClient-Kerberos-Azure" pipeline. +# Kerberos authentication integration tests for Microsoft.Data.SqlClient. The tests consume the +# packages produced by sqlclient-ci-package and run on domain-connected Windows and Linux agents. # -# Schedule: daily at 07:00 UTC on internal/main. -# -# Job breakdown (10 total): -# Stage: windows → 7 jobs (net462 + net8/9/10 × NativeSNI/ManagedSNI) -# Stage: linux → 3 jobs (net8, net9, net10 — ManagedSNI only) -# Stage: Merge-Code-Coverage → 1 job -# -# Shared steps: -# Build and test steps are defined in build-and-test-steps.yml and reused -# by both stages. Each stage passes its OS-specific build target and the -# per-job testFramework matrix variable. -# -# Required ADO variable groups (link in pipeline UI): -# - kv-sqldrivers-shared (provides the agent-at-sqldrv-ad secret) -# -# Variables defined in this file (no pipeline UI configuration needed): -# - KerberosDomain sqldrv.ad -# - KerberosDomainOU agents -# - KerberosDomainUser agent -# - KerberosDomainPassword $(agent-at-sqldrv-ad) from kv-sqldrivers-shared -# - REMOTE_TCP_CONN_STRING TCP connection string to sqldrv-sql22 -# - REMOTE_NP_CONN_STRING Named Pipe connection string to sqldrv-sql22 -# ============================================================================= - -trigger: none # Scheduled runs only — no CI trigger -pr: none # Not triggered by PRs - -schedules: - - cron: '0 7 * * *' - displayName: Daily run (07:00 UTC) - branches: - include: - - internal/main - always: true +# Job breakdown: +# - Windows: net462 with native SNI, plus net8/9/10 with native and managed SNI (7 jobs). +# - Linux: net8/9/10 with managed SNI (3 jobs). name: $(date:yyyyMMdd)$(rev:.r) +# Do not trigger this pipeline for PRs or commits. +trigger: none +pr: none + +# The resource branch filter limits completion triggers to the intended upstream branch. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. +resources: + pipelines: + - pipeline: sqlclient-ci-package + source: sqlclient-ci-package + trigger: + branches: + include: + - internal/main + +parameters: + + - name: buildConfiguration + displayName: Test Build Configuration + type: string + default: Release + values: + - Debug + - Release + + - name: debug + displayName: Enable debug output + type: boolean + default: false + + - name: dotnetVerbosity + displayName: dotnet CLI Verbosity + type: string + default: normal + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + variables: - # Our Kerberos environment doesn't change often, so we can define these variables here rather than - # in the Pipeline UI or in Libraries. + + # Kerberos environment settings shared by all jobs. KerberosDomainPassword resolves from the + # kv-sqldrivers-shared variable group imported by each OS stage. - name: KerberosDomain value: sqldrv.ad @@ -60,7 +68,7 @@ variables: value: agent - name: KerberosDomainPassword - value: $(agent-at-sqldrv-ad) # Secret from kv-sqldrivers-shared variable group + value: $(agent-at-sqldrv-ad) - name: REMOTE_TCP_CONN_STRING value: Data Source=tcp:sqldrv-sql22.sqldrv.ad\sql2022;Initial Catalog=Northwind;Integrated Security=true;Encrypt=false;TrustServerCertificate=true @@ -68,229 +76,9 @@ variables: - name: REMOTE_NP_CONN_STRING value: Data Source=np:sqldrv-sql22.sqldrv.ad\sql2022;Initial Catalog=Northwind;Integrated Security=true;Encrypt=false;TrustServerCertificate=true -# ============================================================================= -# STAGES -# ============================================================================= stages: - - # =========================================================================== - # Stage 1 - Windows (7 jobs = net462 + 3 TFs × 2 ManagedSNI) - # =========================================================================== - - stage: windows - displayName: Windows - dependsOn: [] - variables: - # KerberosDomainPassword expands at runtime in this stage from the agent-at-sqldrv-ad secret - # exposed by this variable group. - - group: kv-sqldrivers-shared - jobs: - - job: windows - displayName: Windows - timeoutInMinutes: 90 - workspace: - clean: all # Purge obj/artifacts from prior runs on self-hosted agents - strategy: - matrix: - # Azure Pipelines exposes matrix variables as environment variables for each step. - # Do not use the name targetFramework here: MSBuild imports environment variables as - # properties, and TargetFramework would leak into dotnet build build.proj, forcing - # transitive project references onto an invalid TFM (for example SqlServer.Server -> net9.0). - net462_NativeSNI: - testFramework: net462 - managedSNI: 'false' - net8_NativeSNI: - testFramework: net8.0 - managedSNI: 'false' - net8_ManagedSNI: - testFramework: net8.0 - managedSNI: 'true' - net9_NativeSNI: - testFramework: net9.0 - managedSNI: 'false' - net9_ManagedSNI: - testFramework: net9.0 - managedSNI: 'true' - net10_NativeSNI: - testFramework: net10.0 - managedSNI: 'false' - net10_ManagedSNI: - testFramework: net10.0 - managedSNI: 'true' - pool: - name: ADO-Trusted-Domain-Win-WestUS2 - demands: - - ImageOverride -equals ADO-MMS22-SQL19 - steps: - - - checkout: self - clean: true - fetchDepth: 1 - fetchTags: false - - - template: /eng/pipelines/common/steps/install-dotnet.yml@self - parameters: - runtimes: [8.x, 9.x] - - # Restore dotnet local tools (pwsh, apicompat, etc.). Required by build.proj targets - # such as _CheckPwshToolRestored that run during the SqlClient ref project build. - - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - - # --- Update test configuration --- - # Uses runtime variables from the matrix ($(managedSNI)) so we use - # inline PowerShell instead of the shared config template which - # requires compile-time parameters. - - pwsh: | - $managedSni = [System.Convert]::ToBoolean($env:MANAGED_SNI) - $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json - foreach ($p in $jdata) { - $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING - $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING - $p.SupportsIntegratedSecurity = $true - $p.UseManagedSNIOnWindows = $managedSni - } - $jdata | ConvertTo-Json | Set-Content "config.jsonc" - workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities - displayName: Update test config.jsonc - env: - REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) - REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) - MANAGED_SNI: $(managedSNI) - - # --- Prepare Windows services --- - - powershell: | - $svc = Get-Service -Name SQLBrowser -ErrorAction SilentlyContinue - if ($null -ne $svc) { - Set-Service -StartupType Automatic SQLBrowser - if ($svc.Status -ne 'Running') { Start-Service SQLBrowser } - Get-Service SQLBrowser | Select-Object Name, StartType, Status - } - displayName: Start SQL Server Browser - - - powershell: | - Set-DtcNetworkSetting -DtcName "Local" ` - -InboundTransactionsEnabled $true ` - -OutboundTransactionsEnabled $true ` - -RemoteClientAccessEnabled $true ` - -Confirm:$false - - Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True - Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC-EPMAP)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True - Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-Out)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True - Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-In)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True - displayName: Enable Network DTC Access - - # --- Build and test --- - - template: /eng/pipelines/ci/kerberos/build-and-test-steps.yml@self - parameters: - testFramework: $(testFramework) - testRunTitle: Windows-$(testFramework)-ManagedSNI_$(managedSNI) - artifactName: $(testFramework)-ManagedSNI_$(managedSNI)-$(System.JobId) - - # =========================================================================== - # Stage 2 - Linux - .NET Core + Kerberos (3 jobs = 3 TFs) - # =========================================================================== - - stage: linux - displayName: Linux - dependsOn: [] - variables: - # KerberosDomainPassword expands at runtime in this stage from the agent-at-sqldrv-ad secret - # exposed by this variable group. - - group: kv-sqldrivers-shared - jobs: - - job: linux - displayName: Linux - timeoutInMinutes: 90 - workspace: - clean: all # Purge leftovers on self-hosted agents to reduce cross-run flakiness - strategy: - matrix: - net8: - testFramework: net8.0 - net9: - testFramework: net9.0 - net10: - testFramework: net10.0 - pool: - name: ADO-Trusted-Linux-WestUS2 - demands: - - ImageOverride -equals ADO-UB20-SQL22 - - steps: - - - checkout: self - clean: true - fetchDepth: 1 - fetchTags: false - - # --- Install .NET SDK and runtimes --- - - template: /eng/pipelines/common/steps/install-dotnet.yml@self - parameters: - runtimes: [8.x, 9.x] - - # Restore dotnet local tools (pwsh, apicompat, etc.). Required by build.proj targets - # such as _CheckPwshToolRestored that run during the SqlClient ref project build. - - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - - # --- Update test configuration (with Kerberos credentials) --- - - pwsh: | - $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json - foreach ($p in $jdata) { - $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING - $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING - $p.SupportsIntegratedSecurity = $true - } - $jdata | Add-Member -NotePropertyName "KerberosDomainUser" -NotePropertyValue $env:KERBEROS_DOMAIN_USER -Force - $jdata | Add-Member -NotePropertyName "KerberosDomainPassword" -NotePropertyValue $env:KERBEROS_DOMAIN_PASSWORD -Force - $jdata | ConvertTo-Json | Set-Content "config.jsonc" - workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities - displayName: Update test config.jsonc (Kerberos) - env: - REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) - REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) - KERBEROS_DOMAIN_USER: $(KerberosDomainUser) - KERBEROS_DOMAIN_PASSWORD: $(KerberosDomainPassword) - - # --- Kerberos domain join --- - - template: /eng/pipelines/ci/kerberos/linux-init-step.yml@self - parameters: - kerberosDomain: $(KerberosDomain) - kerberosDomainOU: $(KerberosDomainOU) - kerberosDomainUser: $(KerberosDomainUser) - kerberosDomainPassword: $(KerberosDomainPassword) - - # --- Verify SQL connectivity --- - - pwsh: | - Install-Module -Name SqlServer -Force -Confirm:$false - Import-Module SqlServer - Invoke-Sqlcmd -Query "SELECT @@VERSION, @@SERVERNAME" -ConnectionString $env:REMOTE_TCP_CONN_STRING - displayName: Verify SQL connectivity - env: - REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) - - # --- Build and test --- - - template: /eng/pipelines/ci/kerberos/build-and-test-steps.yml@self - parameters: - testFramework: $(testFramework) - testRunTitle: Linux-$(testFramework) - artifactName: $(testFramework)-linux-$(System.JobId) - - # --- Kerberos cleanup (always runs) --- - - template: /eng/pipelines/ci/kerberos/linux-cleanup-step.yml@self - parameters: - kerberosDomain: $(KerberosDomain) - kerberosDomainUser: $(KerberosDomainUser) - kerberosDomainPassword: $(KerberosDomainPassword) - - # =========================================================================== - # Stage 3 — Merge Code Coverage (1 job) - # =========================================================================== - - stage: merge - displayName: Merge Code Coverage - dependsOn: - - windows - - linux - condition: succeeded() - jobs: - - template: /eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml@self - parameters: - upload: false + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} diff --git a/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml new file mode 100644 index 0000000000..90269805c9 --- /dev/null +++ b/eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-stages.yml @@ -0,0 +1,107 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Defines separate Windows and Linux Kerberos test stages. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: netFrameworkTestRuntimes + type: object + default: [net462] + + - name: netTestRuntimes + type: object + default: [net8.0, net9.0, net10.0] + +stages: + + - stage: windows + displayName: Windows + dependsOn: [] + variables: + - group: kv-sqldrivers-shared + jobs: + + # .NET Framework uses native SNI. + - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: false + + # .NET runs with both native and managed SNI. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: false + + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Managed SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + poolName: ADO-Trusted-Domain-Win-WestUS2 + runtime: ${{ runtime }} + useManagedSNI: true + + - stage: linux + displayName: Linux + dependsOn: [] + variables: + - group: kv-sqldrivers-shared + jobs: + + # Managed SNI is always used on non-Windows platforms. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/kerberos/sqlclient-ci-kerberos-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Linux : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} + operatingSystem: Linux + poolImage: ADO-UB24 + poolName: ADO-Trusted-Linux-WestUS2 + runtime: ${{ runtime }} diff --git a/eng/pipelines/ci/kerberos/windows-setup-step.yml b/eng/pipelines/ci/kerberos/windows-setup-step.yml new file mode 100644 index 0000000000..9a736f68b9 --- /dev/null +++ b/eng/pipelines/ci/kerberos/windows-setup-step.yml @@ -0,0 +1,52 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Configures a domain-connected Windows agent for Kerberos integration tests. + +parameters: + + - name: useManagedSNI + type: boolean + +steps: + + - pwsh: | + $managedSni = [System.Convert]::ToBoolean($env:MANAGED_SNI) + $jdata = Get-Content -Raw "config.default.jsonc" | ConvertFrom-Json + foreach ($p in $jdata) { + $p.TCPConnectionString = $env:REMOTE_TCP_CONN_STRING + $p.NPConnectionString = $env:REMOTE_NP_CONN_STRING + $p.SupportsIntegratedSecurity = $true + $p.UseManagedSNIOnWindows = $managedSni + } + $jdata | ConvertTo-Json | Set-Content "config.jsonc" + workingDirectory: src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities + displayName: Update test config.jsonc + env: + REMOTE_TCP_CONN_STRING: $(REMOTE_TCP_CONN_STRING) + REMOTE_NP_CONN_STRING: $(REMOTE_NP_CONN_STRING) + MANAGED_SNI: '${{ parameters.useManagedSNI }}' + + - powershell: | + $svc = Get-Service -Name SQLBrowser -ErrorAction SilentlyContinue + if ($null -ne $svc) { + Set-Service -StartupType Automatic SQLBrowser + if ($svc.Status -ne 'Running') { Start-Service SQLBrowser } + Get-Service SQLBrowser | Select-Object Name, StartType, Status + } + displayName: Start SQL Server Browser + + - powershell: | + Set-DtcNetworkSetting -DtcName "Local" ` + -InboundTransactionsEnabled $true ` + -OutboundTransactionsEnabled $true ` + -RemoteClientAccessEnabled $true ` + -Confirm:$false + + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (RPC-EPMAP)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-Out)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + Get-NetFirewallRule -DisplayName "Distributed Transaction Coordinator (TCP-In)" | Set-NetFirewallRule -Profile Domain -Action Allow -Enabled True + displayName: Enable Network DTC Access \ No newline at end of file diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml index 68f59f00c1..3d74f55e0d 100644 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml @@ -72,7 +72,7 @@ parameters: # NOTE: This value is evaluated at template-expansion (compile) time to select the pool image, so # it must be a literal and must not contain any runtime expressions (e.g. $(...) macros or # $[...] runtime expressions). - - name: vmImage + - name: poolImage type: string jobs: @@ -93,7 +93,7 @@ jobs: pool: name: Managed-Instance-pool demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: @@ -104,12 +104,7 @@ jobs: # Download the SqlClient driver packages published by the triggering sqlclient-ci-package # pipeline, stage them into the local NuGet feed, and resolve their exact versions into the # sqlClient/sqlServer/abstractions/logging/azure PackageVersion variables. - # - # sqlServerVersionOverride pins Microsoft.SqlServer.Server to the released stable 1.0.0 to - # avoid an NU1605 downgrade against the >= 1.0.0 dependency from Microsoft.SqlServer.Types. - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self - parameters: - sqlServerVersionOverride: 1.0.0 # Install the .NET SDK and the runtimes needed to execute the test frameworks. - template: /eng/pipelines/common/steps/install-dotnet.yml@self @@ -170,8 +165,6 @@ jobs: -p:TestSet=123 -p:ReferenceType=Package -p:Configuration=${{ parameters.buildConfiguration }} - -p:PackageVersionAbstractions=$(abstractionsPackageVersion) - -p:PackageVersionLogging=$(loggingPackageVersion) -p:PackageVersionSqlClient=$(sqlClientPackageVersion) -p:PackageVersionSqlServer=$(sqlServerPackageVersion) -p:TestResultsFolderPath=TestResults diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml index d579d11994..83acbd34d0 100644 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-pipeline.yml @@ -22,7 +22,9 @@ name: $(DayOfYear)$(Rev:rr) pr: none trigger: none -# Trigger this pipeline after successful runs of the sqlclient-ci-package pipeline. +# The resource branch filter limits completion triggers to the intended upstream branch. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. resources: pipelines: @@ -37,7 +39,10 @@ resources: # added to the project, this resource will fail to resolve and the folder path must be added. - pipeline: sqlclient-ci-package source: sqlclient-ci-package - trigger: true + trigger: + branches: + include: + - internal/main # Pipeline parameters, visible in the Azure DevOps UI. parameters: @@ -76,9 +81,9 @@ parameters: # The stages to run. stages: - # Run the Managed Instance tests. The .NET and .NET Framework runtimes are defaulted by the stage + # Run the Managed Instance tests. The .NET and .NET Framework runtimes are defaulted by the stages # template, which runs both native and managed SNI for .NET on Windows. - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml@self + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml@self parameters: buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml deleted file mode 100644 index 114fef0061..0000000000 --- a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stage.yml +++ /dev/null @@ -1,111 +0,0 @@ -#################################################################################################### -# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this -# file to you under the MIT license. See the LICENSE file in the project root for more information. -#################################################################################################### - -# This stage builds and runs the SqlClient Unit, Functional, and Manual test suites against an Azure -# SQL Managed Instance, building the test projects in "Package" mode against the NuGet packages -# produced by the sqlclient-ci-package pipeline. -# -# It fans out to one job per OS: -# -# - Windows: one native-SNI job per .NET Framework runtime, plus native- and managed-SNI jobs per -# .NET runtime. -# - Linux: one job per .NET runtime (managed SNI is always used on non-Windows). -# -# This template defines a stage named 'managed_instance_tests_stage'. - -parameters: - - # The type of build to produce (Debug or Release). - - name: buildConfiguration - type: string - values: - - Debug - - Release - - # True to enable debugging steps. - - name: debug - type: boolean - - # The verbosity level for the dotnet CLI commands. - - name: dotnetVerbosity - type: string - values: - - quiet - - minimal - - normal - - detailed - - diagnostic - - # The list of .NET (core) runtimes to test against. The same set is used on every OS. - - name: netTestRuntimes - type: object - default: [net8.0, net9.0, net10.0] - - # The list of .NET Framework runtimes to test against on Windows. - - name: netFrameworkTestRuntimes - type: object - default: [net462] - -stages: - - stage: managed_instance_tests_stage - displayName: Run Managed Instance Tests - - jobs: - - # ---------------------------------------------------------------------------------------------- - # Windows: .NET Framework uses native SNI; .NET runs with both native and managed SNI. - - - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - displayName: 'Win : Native SNI : ${{ runtime }}' - jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} - operatingSystem: Windows - runtime: ${{ runtime }} - useManagedSNI: false - vmImage: ADO-MMS22-SQL22 - - - ${{ each runtime in parameters.netTestRuntimes }}: - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayName: 'Win : Native SNI : ${{ runtime }}' - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} - operatingSystem: Windows - runtime: ${{ runtime }} - useManagedSNI: false - vmImage: ADO-MMS22-SQL22 - - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayName: 'Win : Managed SNI : ${{ runtime }}' - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} - operatingSystem: Windows - runtime: ${{ runtime }} - useManagedSNI: true - vmImage: ADO-MMS22-SQL22 - - # ---------------------------------------------------------------------------------------------- - # Linux: one job per .NET runtime (managed SNI is always used on non-Windows). - - - ${{ each runtime in parameters.netTestRuntimes }}: - - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self - parameters: - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayName: 'Linux : ${{ runtime }}' - jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - operatingSystem: Linux - runtime: ${{ runtime }} - vmImage: ADO-UB22-SQL22 diff --git a/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml new file mode 100644 index 0000000000..2176ff7c87 --- /dev/null +++ b/eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-stages.yml @@ -0,0 +1,100 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# Defines separate Windows and Linux stages that run ManualTests against Azure SQL Managed Instance +# using the packages produced by sqlclient-ci-package. + +parameters: + + - name: buildConfiguration + type: string + values: + - Debug + - Release + + - name: debug + type: boolean + + - name: dotnetVerbosity + type: string + values: + - quiet + - minimal + - normal + - detailed + - diagnostic + + - name: netTestRuntimes + type: object + default: [net8.0, net9.0, net10.0] + + - name: netFrameworkTestRuntimes + type: object + default: [net462] + +stages: + + - stage: windows + displayName: Windows + dependsOn: [] + jobs: + + # .NET Framework uses native SNI. + - ${{ each runtime in parameters.netFrameworkTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + displayName: 'Win : Native SNI : ${{ runtime }}' + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + runtime: ${{ runtime }} + useManagedSNI: false + + # .NET runs with both native and managed SNI. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Native SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_native_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + runtime: ${{ runtime }} + useManagedSNI: false + + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Win : Managed SNI : ${{ runtime }}' + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + jobNameSuffix: windows_managed_sni_${{ replace(runtime, '.', '_') }} + operatingSystem: Windows + poolImage: ADO-Win25 + runtime: ${{ runtime }} + useManagedSNI: true + + - stage: linux + displayName: Linux + dependsOn: [] + jobs: + + # Managed SNI is always used on non-Windows platforms. + - ${{ each runtime in parameters.netTestRuntimes }}: + - template: /eng/pipelines/ci/managed-instance/sqlclient-ci-managed-instance-job.yml@self + parameters: + buildConfiguration: ${{ parameters.buildConfiguration }} + debug: ${{ parameters.debug }} + displayName: 'Linux : ${{ runtime }}' + jobNameSuffix: linux_${{ replace(runtime, '.', '_') }} + dotnetVerbosity: ${{ parameters.dotnetVerbosity }} + operatingSystem: Linux + poolImage: ADO-UB24 + runtime: ${{ runtime }} diff --git a/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml b/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml index ada32431b2..41cb134047 100644 --- a/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml +++ b/eng/pipelines/ci/package/sqlclient-ci-package-pipeline.yml @@ -40,8 +40,11 @@ parameters: # The agent image to use for the build. This must exist in both the ADO-1ES-Pool and # ADO-CI-1ES-Pool agent pools. - - name: agentImage - displayName: Agent Image + # + # This keeps a default so that manual runs can pick a Windows agent when that is what we + # want to validate. + - name: poolImage + displayName: Pool Image type: string default: ADO-UB24 values: @@ -76,6 +79,9 @@ parameters: - diagnostic variables: + # Provides 'general_purpose_pool_name', the 1ES pool that hosts most jobs in this project. + - group: sqlclient-pipeline-config-v1 + # Whether this is an internal (ADO.Net project) or public (Public project) build. - name: isInternalBuild value: ${{ eq(variables['System.TeamProject'], 'ADO.Net') }} @@ -93,12 +99,9 @@ jobs: displayName: Build NuGet Packages pool: - ${{ if eq(variables.isInternalBuild, true) }}: - name: ADO-1ES-Pool - ${{ else }}: - name: ADO-CI-1ES-Pool + name: $(general_purpose_pool_name) demands: - - ImageOverride -equals ${{ parameters.agentImage }} + - ImageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml index efd4f233cf..d7a064523e 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-job.yml @@ -71,13 +71,18 @@ parameters: - name: sqlSetupStep type: step - # The pool VM image to use. Both the internal (ADO-1ES-Pool) and public (ADO-CI-1ES-Pool) pools - # must provide an image with this name. + # The name of the pool to run in. # - # NOTE: This value is evaluated at template-expansion (compile) time to select the pool, so it - # must be a literal and must not contain any runtime expressions (e.g. $(...) macros or - # $[...] runtime expressions). - - name: vmImage + # Supplied by the caller so that the pool name flows down from the pipeline root. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. + - name: poolName + type: string + + # The pool VM image to use, which must exist in the specified pool. + - name: poolImage type: string jobs: @@ -89,11 +94,6 @@ jobs: variables: - # Whether this is an internal (ADO.Net project) or public (Public project) build. Evaluated - # at compile time so it can drive template expressions below. - - name: isInternalBuild - value: ${{ eq(variables['System.TeamProject'], 'ADO.Net') }} - # Import the variable group that provides SQL Server build properties used by the # shared configure-sql-server-*-step.yml templates (e.g. x64AliasRegistryPath, # x86AliasRegistryPath, SQLAliasName, SQLAliasPort). @@ -141,7 +141,7 @@ jobs: value: >- -p:ReferenceType=Package -p:SqlClientPackageVersion=$(sqlClientPackageVersion) - -p:AzurePackageVersion=$(azurePackageVersion) + -p:AzurePackageVersion=$(sqlClientPackageVersion) # dotnet CLI options for build. - name: dotnetBuildOpts @@ -162,20 +162,16 @@ jobs: - name: stressTestOpts value: --assembly SqlClient.Stress.Tests --console - # Select the pool based on the requested image: - # - macOS images run on the Microsoft-hosted 'Azure Pipelines' pool (no 1ES macOS images). - # - All other images run on the 1ES pool for the current ADO project, matched via imageOverride. pool: - ${{ if startsWith(parameters.vmImage, 'macos') }}: - name: 'Azure Pipelines' - vmImage: ${{ parameters.vmImage }} + name: ${{ parameters.poolName }} + + # Images provided by Azure Pipelines must be selected using 'vmImage'. + ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: + vmImage: ${{ parameters.poolImage }} + # Images provided by 1ES must be selected using a demand. ${{ else }}: - ${{ if eq(variables.isInternalBuild, true) }}: - name: ADO-1ES-Pool - ${{ else }}: - name: ADO-CI-1ES-Pool demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: @@ -209,14 +205,8 @@ jobs: # Download the SqlClient driver packages published by the triggering sqlclient-ci-package # pipeline, stage them into the local NuGet feed, and resolve their exact versions. The - # sqlClientPackageVersion and azurePackageVersion variables are consumed by the referenceArgs - # variable above. - # - # sqlServerVersionOverride pins Microsoft.SqlServer.Server to the released stable 1.0.0 to - # avoid an NU1605 downgrade against the >= 1.0.0 dependency from Microsoft.SqlServer.Types. + # sqlClientPackageVersion variable is consumed by the referenceArgs variable above. - template: /eng/pipelines/common/steps/download-driver-packages-step.yml@self - parameters: - sqlServerVersionOverride: 1.0.0 # Authenticate with NuGet feeds so that upstream packages (e.g. runtime host packs) can be # fetched through the ADO Artifacts feed. diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml index 74283a8128..813faa09f9 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-pipeline.yml @@ -21,7 +21,9 @@ name: $(DayOfYear)$(Rev:rr) pr: none trigger: none -# Trigger this pipeline after successful runs of the sqlclient-ci-package pipeline. +# The resource branch filters limit completion triggers to the intended upstream branches. Because +# both pipelines use the same repository, an eligible run executes this YAML from the triggering +# package run's branch and commit, preserving branch-specific pipeline definitions. # # The pipeline identifiers are displayed in the Azure DevOps UI, so it is helpful if they indicate # the project, folder, and pipeline name, hence the verbose values below. @@ -44,7 +46,11 @@ resources: # per-project approach instead of this single shared definition). - pipeline: sqlclient-ci-package source: sqlclient-ci-package - trigger: true + trigger: + branches: + include: + - main + - internal/main # Pipeline parameters, visible in the Azure DevOps UI. parameters: @@ -83,6 +89,9 @@ parameters: - detailed - diagnostic +variables: + - template: /eng/pipelines/libraries/ci-build-variables.yml@self + # The stages to run. stages: @@ -90,10 +99,13 @@ stages: - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self parameters: debug: ${{ parameters.debug }} + poolName: $(general_purpose_pool_name) + poolImage: ADO-UB24 # Run the stress tests. - template: /eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml@self parameters: + poolName: $(general_purpose_pool_name) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} warnOnTestFailure: ${{ parameters.warnOnTestFailure }} diff --git a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml index 689d1a702b..6f8ce4aa6d 100644 --- a/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml +++ b/eng/pipelines/ci/stress/sqlclient-ci-stress-stage.yml @@ -20,6 +20,12 @@ parameters: + # The name of the pool to use for jobs that require customized VM images. + # + # Supplied by the caller so that the pool name flows down from the pipeline root. + - name: poolName + type: string + # The type of build to produce (Debug or Release) - name: buildConfiguration type: string @@ -89,7 +95,8 @@ stages: template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self parameters: saPassword: $(saPassword) - vmImage: ADO-UB22-SQL22 + poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24-SQL25 # ---------------------------------------------------------------------------------------------- # Build and test on Windows @@ -112,7 +119,8 @@ stages: saPassword: $(saPassword) # The Windows images include a suitable .NET Framework runtime, so we don't have to install # one explicitly. - vmImage: ADO-MMS22-SQL22 + poolName: ${{ parameters.poolName }} + poolImage: ADO-MMS25-SQL25 # ---------------------------------------------------------------------------------------------- # Build and test on macOS. @@ -133,5 +141,7 @@ stages: template: /eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml@self parameters: saPassword: $(saPassword) - # A macOS vmImage name routes this job to the Microsoft-hosted 'Azure Pipelines' pool. - vmImage: macos-latest + # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted + # 'Azure Pipelines' pool. + poolName: Azure Pipelines + poolImage: macos-latest diff --git a/eng/pipelines/common/steps/align-source-with-upstream-step.yml b/eng/pipelines/common/steps/align-source-with-upstream-step.yml index 11ad2742b7..39249d572d 100644 --- a/eng/pipelines/common/steps/align-source-with-upstream-step.yml +++ b/eng/pipelines/common/steps/align-source-with-upstream-step.yml @@ -36,6 +36,8 @@ steps: - checkout: self clean: true fetchDepth: 0 + fetchTags: false + persistCredentials: true # Align the working-tree source with the commit that built the upstream artifacts, so the projects # compile against the matching (internal) API surface. When the upstream commit is unavailable @@ -43,8 +45,14 @@ steps: - pwsh: | $ErrorActionPreference = 'Stop' $sha = "$(resources.pipeline.${{ parameters.upstreamPipeline }}.sourceCommit)" + if ($sha -notmatch '\A[0-9a-fA-F]{40}\z') { + throw "Invalid ${{ parameters.upstreamPipeline }} commit SHA: '$sha'." + } $pipelineSourceSha = git rev-parse HEAD if ($LASTEXITCODE -ne 0) { throw "Failed to resolve the queued pipeline commit." } + Write-Host "Fetching ${{ parameters.upstreamPipeline }} commit $sha" + git fetch --no-tags origin $sha + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch upstream commit $sha." } Write-Host "Aligning source to ${{ parameters.upstreamPipeline }} commit $sha" git checkout --force $sha if ($LASTEXITCODE -ne 0) { throw "Failed to check out upstream commit $sha." } diff --git a/eng/pipelines/common/steps/download-driver-packages-step.yml b/eng/pipelines/common/steps/download-driver-packages-step.yml index aed4456d31..807affea13 100644 --- a/eng/pipelines/common/steps/download-driver-packages-step.yml +++ b/eng/pipelines/common/steps/download-driver-packages-step.yml @@ -5,14 +5,12 @@ # This template downloads the SqlClient driver packages published by the sqlclient-ci-package # pipeline, copies them into the local NuGet feed (packages/), and resolves the exact package -# versions from the .nupkg filenames. The resolved versions are exposed as job-scoped pipeline -# variables for use by downstream build/test steps: +# versions from the .nupkg filenames. Required SqlClient-family packages are validated to share +# one version. The resolved versions are exposed as job-scoped pipeline variables for use by +# downstream build/test steps: # -# sqlClientPackageVersion Microsoft.Data.SqlClient -# sqlServerPackageVersion Microsoft.SqlServer.Server -# abstractionsPackageVersion Microsoft.Data.SqlClient.Extensions.Abstractions -# loggingPackageVersion Microsoft.Data.SqlClient.Internal.Logging -# azurePackageVersion Microsoft.Data.SqlClient.Extensions.Azure +# sqlClientPackageVersion SqlClient family +# sqlServerPackageVersion Microsoft.SqlServer.Server # # The consuming pipeline MUST declare the triggering pipeline as a resource whose alias matches the # pipelineResource parameter, e.g.: @@ -42,16 +40,6 @@ parameters: type: string default: $(Build.SourcesDirectory)/packages - # Optional override for the Microsoft.SqlServer.Server version. When set, this value is used for - # the sqlServerPackageVersion variable instead of the version resolved from the artifact .nupkg - # filename. Temporary workaround: the CI 'X.Y.Z-ci' prerelease of Microsoft.SqlServer.Server is - # treated as a downgrade of the '>= X.Y.Z' stable dependency that Microsoft.SqlServer.Types pulls - # in (NU1605); passing the released stable version avoids the downgrade. Overall package - # versioning is being addressed separately. - - name: sqlServerVersionOverride - type: string - default: '' - steps: # Download the SqlClient driver packages published by the triggering pipeline into the pipeline @@ -60,8 +48,8 @@ steps: artifact: ${{ parameters.artifactName }} displayName: Download SqlClient Driver Packages - # Copy the downloaded packages into the local NuGet feed and resolve the exact package versions - # from the .nupkg filenames, exposing each as a pipeline variable. + # Copy the downloaded packages into the local NuGet feed, validate the SqlClient family version, + # and expose the two independently versioned package units as pipeline variables. - task: PowerShell@2 displayName: Stage Packages and Resolve Versions inputs: @@ -71,4 +59,3 @@ steps: arguments: >- -FeedPath "${{ parameters.feedPath }}" -ArtifactDirectory "$(Pipeline.Workspace)/${{ parameters.pipelineResource }}/${{ parameters.artifactName }}" - -SqlServerVersionOverride "${{ parameters.sqlServerVersionOverride }}" diff --git a/eng/pipelines/common/steps/download-driver-packages.ps1 b/eng/pipelines/common/steps/download-driver-packages.ps1 index 39f7a38493..4392508991 100644 --- a/eng/pipelines/common/steps/download-driver-packages.ps1 +++ b/eng/pipelines/common/steps/download-driver-packages.ps1 @@ -4,14 +4,12 @@ .DESCRIPTION Copies the .nupkg and .snupkg files downloaded from an upstream SqlClient package artifact into - a local NuGet feed. It then resolves package versions from the .nupkg filenames and emits Azure - Pipelines logging commands that create these job-scoped variables for downstream tasks: + a local NuGet feed. It resolves package versions from the .nupkg filenames, verifies that all + required SqlClient-family packages share one version, and emits Azure Pipelines logging commands + that create these job-scoped variables for downstream tasks: - sqlClientPackageVersion Microsoft.Data.SqlClient - sqlServerPackageVersion Microsoft.SqlServer.Server - abstractionsPackageVersion Microsoft.Data.SqlClient.Extensions.Abstractions - loggingPackageVersion Microsoft.Data.SqlClient.Internal.Logging - azurePackageVersion Microsoft.Data.SqlClient.Extensions.Azure + sqlClientPackageVersion SqlClient family + sqlServerPackageVersion Microsoft.SqlServer.Server Every required .nupkg must be present in ArtifactDirectory. Symbol packages are optional. The script stops on missing required packages, copy failures, and ambiguous filesystem errors. @@ -24,10 +22,6 @@ Directory containing the .nupkg files downloaded from the upstream pipeline artifact. Optional .snupkg files in this directory are copied when present. -.PARAMETER SqlServerVersionOverride - Optional version to expose as sqlServerPackageVersion instead of resolving the version from the - Microsoft.SqlServer.Server .nupkg filename. The package itself is still required and staged. - .EXAMPLE ./download-driver-packages.ps1 ` -FeedPath 'C:\agent\_work\1\s\packages' ` @@ -35,14 +29,6 @@ Stages all driver packages and resolves every version from its package filename. -.EXAMPLE - ./download-driver-packages.ps1 ` - -FeedPath '/agent/_work/1/s/packages' ` - -ArtifactDirectory '/agent/_work/1/sqlclient-ci-package/SqlClient-Driver-Packages' ` - -SqlServerVersionOverride '1.0.0' - - Stages all packages but exposes 1.0.0 as sqlServerPackageVersion. - .OUTPUTS None. Results are emitted as Azure Pipelines task.setvariable logging commands. @@ -61,9 +47,7 @@ param( [string]$FeedPath, [Parameter(Mandatory)] - [string]$ArtifactDirectory, - - [string]$SqlServerVersionOverride = '' + [string]$ArtifactDirectory ) Set-StrictMode -Version Latest @@ -94,47 +78,51 @@ function Resolve-PackageVersion { return [regex]::Match($packages[0].Name, $Pattern).Groups[1].Value } -# Patterns are anchored so the Microsoft.Data.SqlClient package does not also match extension -# packages whose IDs begin with Microsoft.Data.SqlClient. -$packages = @( +# Patterns are anchored so Microsoft.Data.SqlClient does not also match family packages whose IDs +# begin with Microsoft.Data.SqlClient. +$sqlClientFamilyPackages = @( @{ - Variable = 'sqlClientPackageVersion' Name = 'Microsoft.Data.SqlClient' Pattern = '^Microsoft\.Data\.SqlClient\.(\d[^\/]*)\.nupkg$' }, @{ - Variable = 'sqlServerPackageVersion' - Name = 'Microsoft.SqlServer.Server' - Pattern = '^Microsoft\.SqlServer\.Server\.(\d[^\/]*)\.nupkg$' - }, - @{ - Variable = 'abstractionsPackageVersion' Name = 'Microsoft.Data.SqlClient.Extensions.Abstractions' Pattern = '^Microsoft\.Data\.SqlClient\.Extensions\.Abstractions\.(\d[^\/]*)\.nupkg$' }, @{ - Variable = 'loggingPackageVersion' Name = 'Microsoft.Data.SqlClient.Internal.Logging' Pattern = '^Microsoft\.Data\.SqlClient\.Internal\.Logging\.(\d[^\/]*)\.nupkg$' }, @{ - Variable = 'azurePackageVersion' Name = 'Microsoft.Data.SqlClient.Extensions.Azure' Pattern = '^Microsoft\.Data\.SqlClient\.Extensions\.Azure\.(\d[^\/]*)\.nupkg$' + }, + @{ + Name = 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' + Pattern = '^Microsoft\.Data\.SqlClient\.AlwaysEncrypted\.AzureKeyVaultProvider\.(\d[^\/]*)\.nupkg$' } ) -foreach ($package in $packages) { - $artifactVersion = Resolve-PackageVersion $ArtifactDirectory $package.Pattern $package.Name +$sqlClientPackageVersion = Resolve-PackageVersion ` + $ArtifactDirectory ` + $sqlClientFamilyPackages[0].Pattern ` + $sqlClientFamilyPackages[0].Name - if ($package.Variable -eq 'sqlServerPackageVersion' -and - -not [string]::IsNullOrWhiteSpace($SqlServerVersionOverride)) { - $version = $SqlServerVersionOverride - Write-Host "Overriding $($package.Name) version from $artifactVersion to $version" - } else { - $version = $artifactVersion - Write-Host "Resolved $($package.Name) version: $version" +foreach ($package in $sqlClientFamilyPackages) { + $version = Resolve-PackageVersion $ArtifactDirectory $package.Pattern $package.Name + + if ($version -ne $sqlClientPackageVersion) { + throw "$($package.Name) version $version does not match SqlClient family version $sqlClientPackageVersion" } - Write-Host "##vso[task.setvariable variable=$($package.Variable)]$version" + Write-Host "Validated $($package.Name) version: $version" } + +$sqlServerPackageVersion = Resolve-PackageVersion ` + $ArtifactDirectory ` + '^Microsoft\.SqlServer\.Server\.(\d[^\/]*)\.nupkg$' ` + 'Microsoft.SqlServer.Server' + +Write-Host "Resolved Microsoft.SqlServer.Server version: $sqlServerPackageVersion" +Write-Host "##vso[task.setvariable variable=sqlClientPackageVersion]$sqlClientPackageVersion" +Write-Host "##vso[task.setvariable variable=sqlServerPackageVersion]$sqlServerPackageVersion" diff --git a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml index 95d5d0b7d0..9b7a88e6cc 100644 --- a/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml @@ -19,15 +19,17 @@ parameters: # Reference sibling packages as C# projects. - Project - # The name of Azure Pipelines pool to use. + # The name of the 1ES pool to use. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # - name: poolName type: string - default: $(ci_var_defaultPoolName) - # The name of the Azure Pipelines image to use within the pool. - - name: imageOverride + # The name of the VM image to run on, within the pool. + - name: poolImage type: string - default: ADO-MMS22-SQL19 # The name of the Abstractions pipeline artifact to download when referenceType is 'Package'. - name: abstractionsArtifactsName @@ -83,10 +85,7 @@ jobs: pool: name: ${{parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.imageOverride }} - - variables: - - template: /eng/pipelines/libraries/ci-build-variables.yml@self + - imageOverride -equals ${{ parameters.poolImage }} steps: - ${{ if eq(parameters.debug, true)}}: diff --git a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml index 95c0ac0f3c..0b71bbaa5e 100644 --- a/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml @@ -19,13 +19,27 @@ parameters: - name: upload type: boolean + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: publish_code_coverage displayName: Publish Code Coverage pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: # Use a temp directory that is cleaned up after each job runs. This helps @@ -112,12 +126,13 @@ jobs: displayName: '[Debug] Show Disk Usage' # Publish the Cobertura XML coverage file as a pipeline artifact for - # debugging purposes. + # debugging purposes. Pipeline artifact names must be unique within a + # build, so include the attempt numbers to allow this job to be rerun. - task: PublishPipelineArtifact@1 displayName: Publish Cobertura XML Artifact inputs: targetPath: $(workingDir)/merge - artifact: Cobertura Merge Results + artifact: Cobertura Merge Results $(System.StageAttempt)-$(System.JobAttempt) # Publish the Cobertura reports to the pipeline to be viewed in the Azure # DevOps pipeline run UI. diff --git a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml index 4752e09e43..86ee3806fd 100644 --- a/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml +++ b/eng/pipelines/common/templates/jobs/ci-run-tests-job.yml @@ -56,14 +56,8 @@ parameters: type: boolean default: false - # True if this job will run in a generic Azure Pipelines hosted pool; false to run in a custom 1ES - # pool. - - name: hostedPool - type: boolean - default: false - # The VM image to use, which must exist in the specified pool. - - name: image + - name: poolImage type: string # The display name for this job. @@ -110,6 +104,10 @@ parameters: - Release # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string @@ -149,7 +147,7 @@ parameters: type: string jobs: -- job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.image, 'unknown_image')) }} +- job: ${{ format('{0}', coalesce(parameters.jobDisplayName, parameters.poolImage, 'unknown_image')) }} # Some of our tests take longer than the default 60 minutes to run on some # OSes and configurations. @@ -157,11 +155,14 @@ jobs: pool: name: '${{ parameters.poolName }}' - ${{ if eq(parameters.hostedPool, true) }}: - vmImage: ${{ parameters.image }} + + # Images provided by Azure Pipelines must be selected using 'vmImage'. + ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: + vmImage: ${{ parameters.poolImage }} + # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.image }} + - imageOverride -equals ${{ parameters.poolImage }} variables: - name: dotnetx86RootPath diff --git a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml index 7ed67c1af9..29e0042569 100644 --- a/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml +++ b/eng/pipelines/common/templates/stages/ci-run-tests-stage.yml @@ -106,8 +106,7 @@ stages: referenceType: ${{ parameters.referenceType }} timeout: ${{ parameters.testJobTimeout }} poolName: ${{ config.value.pool }} - hostedPool: ${{ eq(config.value.hostedPool, true) }} - image: ${{ image.value }} + poolImage: ${{ image.value }} jobDisplayName: ${{ format('{0}_{1}_{2}', replace(targetFramework, '.', '_'), platform, testSet) }} configProperties: ${{ config.value.configProperties }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} @@ -137,8 +136,7 @@ stages: referenceType: ${{ parameters.referenceType }} timeout: ${{ parameters.testJobTimeout }} poolName: ${{ config.value.pool }} - hostedPool: ${{ eq(config.value.hostedPool, true) }} - image: ${{ image.value }} + poolImage: ${{ image.value }} ${{if eq(usemanagedSNI, 'true') }}: jobDisplayName: ${{ format('{0}_{1}_{2}_{3}', replace(targetFramework, '.', '_'), platform, 'ManagedSNI', testSet) }} ${{ else }}: diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml index 15de459d50..bafb9eda1c 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml @@ -4,8 +4,8 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This step configures an existing SQL Server running on the local Linux host. For example, our 1ES -# Hosted Pool has images like ADO-UB20-SQL22 that come with SQL Server 2022 pre-installed and +# This step configures an existing SQL Server running on the local Linux host. For example, our +# 1ES pools have images like ADO-UB24-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: @@ -16,8 +16,16 @@ parameters: steps: + # Make sure sqlcmd is available; the SQL Server images do not always ship it. + - template: /eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml@self + # Configure SQL Server. - bash: | + if [ -z "${SQLCMD_BIN:-}" ]; then + echo "ERROR: sqlcmd was not resolved by the 'Install sqlcmd [Linux]' step." + exit 1 + fi + sudo systemctl stop mssql-server # Password for the SA user (required) @@ -39,10 +47,11 @@ steps: do echo Waiting for SQL Server to start... sleep 3s - /opt/mssql-tools/bin/sqlcmd \ + "$SQLCMD_BIN" \ -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -Q "SELECT @@VERSION" 2>/dev/null errstatus=$? ((counter++)) @@ -55,3 +64,6 @@ steps: exit $errstatus fi displayName: 'Configure SQL Server [Linux]' + env: + SQLCMD_BIN: $(SqlCmdBin) + SQLCMD_TRUST_ARG: $(SqlCmdTrustArg) diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml index 89b5e9f738..0ffeb4f2ab 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml @@ -29,21 +29,33 @@ steps: export PS4='+ [$(date "+%Y-%m-%d %H:%M:%S")] ' set -x - # Install Docker CLI (not Desktop — Colima provides the daemon) and SQLCMD tools. + # Install Colima (which provides the Docker daemon, since Docker Desktop is + # not available here), the docker CLI, and sqlcmd. brew install colima - brew install docker - brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release - brew update - # Homebrew 5.2+ requires explicit trust for third-party taps. Run this - # after 'brew update' so the trust command is available even if the runner - # image shipped an older Homebrew version. - brew trust microsoft/mssql-release - HOMEBREW_ACCEPT_EULA=Y brew install mssql-tools18 - - # Fail fast if sqlcmd was not installed (e.g. tap-trust or formula error). - # Without this check the script would loop for ~6 minutes trying to connect. + + # Homebrew ships no Intel macOS bottle for the current docker formula, so + # 'brew install docker' compiles the CLI (and builds Go to do it), which + # takes minutes and often exhausts the step timeout. Install the newest + # bottled version directly instead. + DOCKER_CLI_DIR="$HOME/.docker-cli/bin" + if ! pwsh -NoProfile -File "$(Build.SourcesDirectory)/eng/pipelines/scripts/Install-DockerCli.macos.ps1" -DestinationPath "$DOCKER_CLI_DIR"; then + echo "ERROR: Failed to install the docker CLI." + exit 1 + fi + # prependpath only affects later steps, so fix PATH for this one as well. + export PATH="$DOCKER_CLI_DIR:$PATH" + + # go-sqlcmd, rather than mssql-tools18 from the microsoft/mssql-release + # tap. That tap pins an openssl@3 formula that has no Intel bottle, so + # installing it compiled openssl from source and cost 12.5 minutes of the + # step budget. go-sqlcmd is a single bottled Go binary with no openssl + # dependency, and accepts the same flags used below. + brew install sqlcmd + + # Fail fast if sqlcmd was not installed. Without this check the script + # would loop for ~6 minutes trying to connect. if ! command -v sqlcmd &>/dev/null; then - echo "ERROR: sqlcmd is not on PATH after brew install. Check the mssql-tools18 installation above." + echo "ERROR: sqlcmd is not on PATH after 'brew install sqlcmd'." exit 1 fi @@ -82,8 +94,8 @@ steps: fi # Point the docker CLI at Colima's daemon socket. Colima normally sets an - # active docker context, but the standalone docker CLI (installed above via - # 'brew install docker') can default to unix:///var/run/docker.sock, which + # active docker context, but the standalone docker CLI installed above can + # default to unix:///var/run/docker.sock, which # does not exist on macOS without Docker Desktop. This caused every # 'docker pull' to fail instantly with: # failed to connect to the docker API at unix:///var/run/docker.sock @@ -129,84 +141,143 @@ steps: # Password for the SA user (required) MSSQL_SA_PW="${{ parameters.saPassword }}" - # Start the container and fail fast (with diagnostics) if it does not launch, - # rather than silently falling through to the connection-wait loop. - if ! docker run --platform linux/amd64 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2025-latest; then - echo "ERROR: Failed to start the sql1 container." - docker ps -a --filter "name=^/sql1$" - exit 1 - fi + # Collect everything needed to diagnose a container that crashed or never + # became ready. + # + # The mssql dump collector (paldumper) writes hundreds of + # "find: '/proc/N/...': Permission denied" lines to the container's stderr + # whenever sqlservr faults. That noise used to fill the entire captured + # window and hide the actual SQL Server error, so it is filtered out here + # and both the head and the tail of the log are printed. + dumpSqlDiagnostics() + { + set +x - sleep 10 + echo "--- Container status ---" + docker ps -a --filter "name=^/sql1$" || true - docker ps -a + echo "--- Container state ---" + docker inspect sql1 --format 'ExitCode={{.State.ExitCode}} OOMKilled={{.State.OOMKilled}} Error="{{.State.Error}}" StartedAt={{.State.StartedAt}} FinishedAt={{.State.FinishedAt}}' || true - # Connect to the SQL Server container and get its version. - # - # With Rosetta 2 emulation, SQL Server starts much faster than under full - # QEMU emulation, but it can still take a minute or two. We allow up to - # 6 minutes (72 attempts × 5 seconds) as a generous upper bound. + echo "--- SQL Server errorlog (last 80 lines) ---" + # docker cp works against a stopped container, so this survives a crash. + if docker cp sql1:/var/opt/mssql/log/errorlog "$SQL_ERRORLOG" 2>/dev/null; then + tail -80 "$SQL_ERRORLOG" + else + echo "(errorlog not available)" + fi + + echo "--- Container logs, dump-collector noise filtered (first 60 lines) ---" + docker logs sql1 2>&1 | grep -vE "^(find|dmesg|timeout): " | head -60 || true + echo "--- Container logs, dump-collector noise filtered (last 40 lines) ---" + docker logs sql1 2>&1 | grep -vE "^(find|dmesg|timeout): " | tail -40 || true + + echo "--- sqlcmd errors ---" + cat "$SQLCMD_ERRORS" 2>/dev/null || echo "(none)" - # Wait 5 seconds between attempts. - delay=5 + echo "--- Host capacity ---" + echo "hw.ncpu=$(sysctl -n hw.ncpu 2>/dev/null) hw.memsize=$(sysctl -n hw.memsize 2>/dev/null)" + vm_stat || true - # Try up to 72 times (~6 minutes) to connect. - maxAttempts=72 + echo "--- Guest capacity ---" + colima ssh -- free -m || true - # Attempt counter. - attempt=1 + set -x + } - # Flag to indicate when SQL Server is ready to accept connections. - ready=0 + SQL_ERRORLOG=$(Agent.TempDirectory)/mssql_errorlog - while [ $attempt -le $maxAttempts ] + # Start SQL Server, retrying the whole container lifecycle. sqlservr + # intermittently core dumps within a minute or two of starting on these + # agents (the container goes to "Exited (1)" while the readiness loop is + # still polling), which previously failed the entire step on the first + # occurrence. Colima startup and the image pull already retry; the + # container did not. + runAttempts=3 + sqlReady=0 + + for ((r=1; r<=runAttempts; r++)) do + echo "Starting SQL Server container (attempt #$r of $runAttempts)..." + + # Clear any container and sqlcmd output left behind by a prior attempt. + docker rm -f sql1 >/dev/null 2>&1 || true + : > "$SQLCMD_ERRORS" + + if ! docker run --platform linux/amd64 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2025-latest; then + echo "ERROR: Failed to start the sql1 container (attempt #$r of $runAttempts)." + dumpSqlDiagnostics + continue + fi - echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." + sleep 10 - # -C trusts the self-signed certificate inside the container. - sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" >> $SQLCMD_ERRORS 2>&1 + docker ps -a - # If the command was successful, then the SQL Server is ready. - if [ $? -eq 0 ]; then - ready=1 + # Connect to the SQL Server container and get its version. + # + # With Rosetta 2 emulation, SQL Server starts much faster than under full + # QEMU emulation, but it can still take a minute or two. We allow up to + # 6 minutes (72 attempts × 5 seconds) as a generous upper bound. + + # Wait 5 seconds between attempts. + delay=5 + + # Try up to 72 times (~6 minutes) to connect. + maxAttempts=72 + + # Attempt counter. + attempt=1 + + # Set when the container died before SQL Server accepted connections. + crashed=0 + + while [ $attempt -le $maxAttempts ] + do + + echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..." + + # -C trusts the self-signed certificate inside the container. + if sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" >> "$SQLCMD_ERRORS" 2>&1; then + sqlReady=1 + break + fi + + # Verify the container is still running; no point retrying if it crashed. + if ! docker ps --filter "name=^/sql1$" --filter "status=running" --format '{{.Names}}' | grep -Fxq 'sql1'; then + echo "ERROR: sql1 container is no longer running (attempt #$r of $runAttempts)." + crashed=1 + break + fi + + # Increment the attempt counter. + ((attempt++)) + + # Wait before trying again. + sleep $delay + + done + + if [ $sqlReady -eq 1 ]; then break fi - # Verify the container is still running; no point retrying if it crashed. - if ! docker ps --filter "name=^/sql1$" --filter "status=running" --format '{{.Names}}' | grep -Fxq 'sql1'; then - echo "ERROR: sql1 container is no longer running." - docker ps -a --filter "name=^/sql1$" - echo "--- Container logs ---" - docker logs sql1 2>&1 | tail -50 - rm -f $SQLCMD_ERRORS - exit 1 + if [ $crashed -ne 1 ]; then + echo "ERROR: Cannot connect to SQL Server after $maxAttempts attempts (attempt #$r of $runAttempts)." fi - # Increment the attempt counter. - ((attempt++)) - - # Wait before trying again. - sleep $delay - + dumpSqlDiagnostics done # Is the SQL Server ready? - if [ $ready -eq 0 ] + if [ $sqlReady -ne 1 ] then - # No, so report the error(s) and exit. - echo "Cannot connect to SQL Server after $maxAttempts attempts; installation aborted." - echo "--- sqlcmd errors ---" - cat $SQLCMD_ERRORS - echo "--- Container status ---" - docker ps -a --filter "name=^/sql1$" - echo "--- Container logs (last 80 lines) ---" - docker logs sql1 2>&1 | tail -80 - rm -f $SQLCMD_ERRORS + echo "ERROR: SQL Server did not become ready after $runAttempts container attempts; installation aborted." + rm -f "$SQLCMD_ERRORS" "$SQL_ERRORLOG" exit 1 fi - rm -f $SQLCMD_ERRORS + rm -f "$SQLCMD_ERRORS" "$SQL_ERRORLOG" echo "Use sqlcmd to show which IP addresses are being listened on..." echo 0.0.0.0 @@ -231,3 +302,10 @@ steps: fi displayName: 'Configure SQL Server [macOS]' + # Well above a healthy run, but low enough that a wedged install or Colima + # boot fails here instead of consuming the whole test job. Measured worst + # case is ~24 minutes: Colima boot ~6, the SQL image pull ~10 (7 of which is + # extraction inside the VM), and up to 6 more waiting for SQL to accept + # connections. The container is now started up to 3 times, so the readiness + # wait can cost 18 minutes rather than 6 in the pathological case. + timeoutInMinutes: 55 diff --git a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml index 5659d45de2..cd691cc4a2 100644 --- a/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml +++ b/eng/pipelines/common/templates/steps/configure-sql-server-win-step.yml @@ -5,7 +5,7 @@ ################################################################################# # This step configures an existing SQL Server running on the local Windows host. For example, our -# 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come with SQL Server 2022 pre-installed and +# 1ES pools have images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: diff --git a/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml b/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml new file mode 100644 index 0000000000..d5e82f434b --- /dev/null +++ b/eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml @@ -0,0 +1,83 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Ensures the SQL Server command line tools are available on a Linux agent. +# +# The SQL Server images in our 1ES pool (for example ADO-UB24-SQL25) ship the SQL Server engine but +# do not always ship the command line tools, so install them when they are missing. +# +# Sets two variables for the remaining steps in the job: +# +# SqlCmdBin - absolute path to the sqlcmd executable. +# SqlCmdTrustArg - '-C' when sqlcmd needs to be told to trust the server's self-signed +# certificate, otherwise empty. + +steps: + + - bash: | + set -u + + if ! command -v sudo >/dev/null 2>&1; then + echo "ERROR: 'sudo' is required to install the SQL Server command line tools." + exit 1 + fi + + find_sqlcmd() { + if command -v sqlcmd >/dev/null 2>&1; then + command -v sqlcmd + return 0 + fi + # mssql-tools default install locations. + local candidate + for candidate in /opt/mssql-tools18/bin/sqlcmd /opt/mssql-tools/bin/sqlcmd; do + if [ -x "$candidate" ]; then + echo "$candidate" + return 0 + fi + done + return 1 + } + + if SQLCMD_BIN="$(find_sqlcmd)"; then + echo "Found existing sqlcmd at '$SQLCMD_BIN'." + else + echo "sqlcmd was not found; installing mssql-tools18..." + + # The Microsoft package repository is usually already configured on these images (that is + # where the engine came from), so try a plain install first and only register the repository + # if that fails. + sudo apt-get update + if ! sudo env ACCEPT_EULA=Y apt-get install -y mssql-tools18 unixodbc-dev; then + # shellcheck disable=SC1091 + . /etc/os-release + echo "Registering the Microsoft package repository for Ubuntu $VERSION_ID..." + curl -sSL -o /tmp/packages-microsoft-prod.deb \ + "https://packages.microsoft.com/config/ubuntu/$VERSION_ID/packages-microsoft-prod.deb" + sudo dpkg -i /tmp/packages-microsoft-prod.deb + sudo apt-get update + sudo env ACCEPT_EULA=Y apt-get install -y mssql-tools18 unixodbc-dev + fi + + if ! SQLCMD_BIN="$(find_sqlcmd)"; then + echo "ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations," + echo " and installing mssql-tools18 did not provide it." + exit 1 + fi + fi + + # sqlcmd from mssql-tools18 (and go-sqlcmd) encrypts by default, so it needs -C to trust the + # local server's self-signed certificate. The older mssql-tools build does not support -C. + case "$SQLCMD_BIN" in + */mssql-tools/bin/sqlcmd) SQLCMD_TRUST_ARG="" ;; + *) SQLCMD_TRUST_ARG="-C" ;; + esac + + echo "Using sqlcmd '$SQLCMD_BIN' with trust argument '$SQLCMD_TRUST_ARG'." + + # Publish for the remaining steps in this job. + echo "##vso[task.setvariable variable=SqlCmdBin]$SQLCMD_BIN" + echo "##vso[task.setvariable variable=SqlCmdTrustArg]$SQLCMD_TRUST_ARG" + displayName: 'Install sqlcmd [Linux]' diff --git a/eng/pipelines/common/templates/steps/override-sni-version.yml b/eng/pipelines/common/templates/steps/override-sni-version.yml index 3b275262c3..b3496b9f1f 100644 --- a/eng/pipelines/common/templates/steps/override-sni-version.yml +++ b/eng/pipelines/common/templates/steps/override-sni-version.yml @@ -42,6 +42,19 @@ steps: # add the new package source $packageSources.AppendChild($newSource) + # Exact mappings take precedence over the governed feed's wildcard, ensuring validation SNI + # packages are restored from this source. Both package IDs are externally produced and are + # therefore intentionally not eligible for the repository's local feed. + $packageSourceMapping = $xml.SelectSingleNode('//ns:packageSourceMapping', $nsm) + $newMapping = $xml.CreateElement("packageSource") + $newMapping.SetAttribute("key","SNIValidation") + foreach ($packageId in @("Microsoft.Data.SqlClient.SNI", "Microsoft.Data.SqlClient.SNI.runtime")) { + $package = $xml.CreateElement("package") + $package.SetAttribute("pattern", $packageId) + $newMapping.AppendChild($package) + } + $packageSourceMapping.AppendChild($newMapping) + # save the xml file $xml.Save($NugetCfg) type $NugetCfg diff --git a/eng/pipelines/common/templates/steps/publish-test-results-step.yml b/eng/pipelines/common/templates/steps/publish-test-results-step.yml index 9cb356aa45..8e06e9dece 100644 --- a/eng/pipelines/common/templates/steps/publish-test-results-step.yml +++ b/eng/pipelines/common/templates/steps/publish-test-results-step.yml @@ -59,9 +59,22 @@ steps: Get-ChildItem -Filter "*.coverage" -Recurse displayName: '[Debug] List test result coverage files' +# When an earlier step fails (e.g. SQL Server setup), no TestResults directory is +# produced and PublishPipelineArtifact fails with "Path does not exist", adding a +# second error that buries the real one. Gate the publish on the directory +# actually existing. +- pwsh: | + $exists = Test-Path -Path 'TestResults' -PathType Container + if (-not $exists) { + Write-Host 'No TestResults directory was produced; skipping test artifact publish.' + } + Write-Host "##vso[task.setvariable variable=HasTestResults]$($exists.ToString().ToLowerInvariant())" + displayName: 'Check for test results' + condition: succeededOrFailed() + - task: PublishPipelineArtifact@1 displayName: 'Publish Test Artifacts' inputs: targetPath: TestResults artifact: '${{parameters.targetFramework }}WinAz$(System.JobId)' - condition: succeededOrFailed() + condition: and(succeededOrFailed(), eq(variables['HasTestResults'], 'true')) diff --git a/eng/pipelines/dotnet-sqlclient-ci-core.yml b/eng/pipelines/dotnet-sqlclient-ci-core.yml index 3e68e93325..8910ccac33 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-core.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-core.yml @@ -25,6 +25,26 @@ parameters: type: object default: [net8.0, net9.0, net10.0] + # The target frameworks to build and run tests for on Windows, for the + # primary test configurations (local SQL Server 2025 and Azure SQL). + # + # These configurations carry the broadest coverage, so newer runtimes are + # validated here first before being enabled across every configuration. + # + # Note: The driver does not ship a net10.0 target framework, so net10.0 test + # assemblies resolve the net9.0 driver build. Including net10.0 here + # validates the driver running on the .NET 10 runtime. + # + - name: primaryTargetFrameworks + type: object + default: [net462, net8.0, net9.0, net10.0] + + # The target frameworks to build and run tests for on Unix, for the primary + # test configurations (local SQL Server 2025 and Azure SQL). + - name: primaryTargetFrameworksUnix + type: object + default: [net8.0, net9.0, net10.0] + # Netcore Version for Test Utilities - name: netcoreVersionTestUtils type: object @@ -74,9 +94,21 @@ parameters: - Debug - Release + # The name of the general-purpose 1ES pool that most CI jobs run in. + # + # This is the single place in the CI pipelines where the pool name is read + # from a variable group; every stage and job below receives it as a + # parameter so that the value flows down from here. + # + # 'general_purpose_pool_name' is defined in the 'sqlclient-pipeline-config-v1' + # variable group (see /eng/pipelines/libraries/ci-build-variables.yml), for + # both the Public and ADO.Net projects, and names each project's own pool. + # Special-purpose pools, such as the Always Encrypted and ARM64 pools used + # below, are still named directly. + # - name: defaultPoolName type: string - default: $(ci_var_defaultPoolName) + default: $(general_purpose_pool_name) # The timeout, in minutes, for each test job. - name: testJobTimeout @@ -94,6 +126,15 @@ parameters: type: boolean default: true + # If true, run manual tests against SQL Server 2022. + # + # The primary test configurations run against SQL Server 2025, so SQL Server + # 2022 is CI-only coverage. PR pipelines disable it to keep validation fast. + # + - name: runSql22Tests + type: boolean + default: true + # Build suffix appended to the prerelease tag. PR pipelines pass 'pr', # CI pipelines pass 'ci'. Official builds leave this empty. - name: buildSuffix @@ -128,16 +169,19 @@ variables: value: SqlServer.Artifacts stages: - # Compute all package versions up-front. Downstream stages consume these via - # Each build/test stage computes the versions it needs directly from the - # compute_versions_ci stage outputs, so no version parameters are passed here. + # Compute all package versions up front. Build and test stages consume the + # compute_versions_ci outputs directly, so no version parameters are passed here. - template: /eng/pipelines/stages/compute-versions-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 buildSuffix: ${{ parameters.buildSuffix }} # Generate secrets used throughout the pipeline. - template: /eng/pipelines/stages/generate-secrets-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 debug: ${{ parameters.debug }} # Build the SqlServer package, and publish it to the pipeline artifacts @@ -145,6 +189,7 @@ stages: # generation stage since it has no package dependencies. - template: /eng/pipelines/stages/build-sqlserver-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} sqlServerArtifactsName: $(sqlServerArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} @@ -155,6 +200,7 @@ stages: # generation stage since it has no package dependencies. - template: /eng/pipelines/stages/build-logging-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} loggingArtifactsName: $(loggingArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} @@ -167,6 +213,7 @@ stages: # Abstractions has a package dependency on Logging. - template: /eng/pipelines/stages/build-abstractions-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} @@ -185,6 +232,7 @@ stages: # - template: /eng/pipelines/stages/build-sqlclient-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} loggingArtifactsName: $(loggingArtifactsName) @@ -205,6 +253,7 @@ stages: # given artifact name. - template: /eng/pipelines/stages/build-azure-package-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} abstractionsArtifactsName: $(abstractionsArtifactsName) azureArtifactsName: $(azureArtifactsName) buildConfiguration: ${{ parameters.buildConfiguration }} @@ -228,6 +277,8 @@ stages: # published as pipeline artifacts. - template: /eng/pipelines/stages/verify-nuget-packages-ci-stage.yml@self parameters: + poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-Win25 abstractionsArtifactsName: $(abstractionsArtifactsName) azureArtifactsName: $(azureArtifactsName) loggingArtifactsName: $(loggingArtifactsName) @@ -275,6 +326,8 @@ stages: - template: /eng/pipelines/common/templates/jobs/ci-code-coverage-job.yml@self parameters: debug: ${{ parameters.debug }} + poolName: ${{ parameters.defaultPoolName }} + poolImage: ADO-UB24 # We only want to upload coverage results to CodeCov from certain # pipelines. We use the pipeline name (Build.DefinitionName) to # choose. This is a predefined variable that is available at @@ -296,13 +349,14 @@ stages: # Configuration of test jobs. Each entry in this object will become a test job, and the # properties of each entry will be supplied as parameters to the test job template. + # + # The OS architecture is assumed to be x64 unless otherwise noted. + # testConfigurations: - # SQL Server 2016 and 2017 on Windows Server 2022 (x64 only). - # x86 testing is intentionally skipped for these legacy SQL versions - # because x86 support is already validated via SQL 2019 and 2022 images. + # SQL Server 2016 and 2017 on Windows Server 2022. ${{ if eq(parameters.runLegacySqlTests, true) }}: - # Windows Server 22 with local SQL Server 2016, x64 build platform. - windows_sql_16_x64: + # Windows Server 22 with local SQL Server 2016. + win22_sql16: pool: ${{parameters.defaultPoolName }} images: Win22_Sql16: ADO-MMS22-SQL16 @@ -327,8 +381,8 @@ stages: SQLRootPath: $(SQL16RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2017, x64 build platform. - windows_sql_17_x64: + # Windows Server 22 with local SQL Server 2017. + win22_sql17: pool: ${{parameters.defaultPoolName }} images: Win22_Sql17: ADO-MMS22-SQL17 @@ -353,8 +407,8 @@ stages: SQLRootPath: $(SQL17RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2019, x64 build platform. - windows_sql_19_x64: + # Windows Server 22 with local SQL Server 2019. + win22_sql19: pool: ${{parameters.defaultPoolName }} images: Win22_Sql19: ADO-MMS22-SQL19 @@ -380,8 +434,8 @@ stages: SQLRootPath: $(SQL19RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2019, x86 build platform. - windows_sql_19_x86: + # Windows Server 22 with local SQL Server 2019, x86. + win22_sql19_x86: pool: ${{parameters.defaultPoolName }} images: Win22_Sql19_x86: ADO-MMS22-SQL19 @@ -407,12 +461,12 @@ stages: SQLRootPath: $(SQL19RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022, x64 build platform. - windows_sql_22_x64: + # Windows Server 25 with local SQL Server 2025. + win25_sql25: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22: ADO-MMS22-SQL22 - TargetFrameworks: ${{parameters.targetFrameworks }} + Win25_Sql25: ADO-MMS25-SQL25 + TargetFrameworks: ${{parameters.primaryTargetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} testSets: ${{parameters.testSets }} @@ -430,14 +484,14 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022, x86 build platform. - windows_sql_22_x86: + # Windows Server 25 with local SQL Server 2025, x86. + win25_sql25_x86: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22_x86: ADO-MMS22-SQL22 + Win25_Sql25_x86: ADO-MMS25-SQL25 TargetFrameworks: [net462, net8.0, net9.0] netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -457,14 +511,14 @@ stages: LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) x86TestTargetFrameworks: [net462, net8.0, net9.0] - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) enableLocalDB: true - # Windows Server 22 with local SQL Server 2022 Named Instance, x64 build platform. - windows_sql_22_named_instance: + # Windows Server 25 with local SQL Server 2025 Named Instance. + win25_sql25_named_instance: pool: ${{parameters.defaultPoolName }} images: - Win22_Sql22_Named_Instance: ADO-MMS22-SQL22-WITH-NAMED-INSTANCE + Win25_Sql25_Named_Instance: ADO-MMS25-SQL25-WITH-NAMED-INSTANCE TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -476,16 +530,15 @@ stages: TCPConnectionString: $(SQL_TCP_INSTANCE_CONN_STRING) NPConnectionString: $(SQL_NP_INSTANCE_CONN_STRING) SupportsIntegratedSecurity: true - SQLRootPath: $(SQL22RootPath) + SQLRootPath: $(SQL25RootPath) instanceName: $(NamedInstance) - # Windows Server 2022 and Windows 11, x64 build platform, with Azure SQL Server. - windows_azure_sql: + # Windows Server 2025, x64 build platform, with Azure SQL Server. + win25_azure_sql: pool: ${{parameters.defaultPoolName }} images: - Win22_Azure_Sql: ADO-MMS22-SQL19 - Win11_Azure_Sql: ADO-CI-Win11 - TargetFrameworks: ${{parameters.targetFrameworks }} + Win25_Azure_Sql: ADO-Win25 + TargetFrameworks: ${{parameters.primaryTargetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} testSets: ${{parameters.testSets }} @@ -510,11 +563,38 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + # Windows 11, x64 build platform, with Azure SQL Server. + win11_azure_sql: + pool: ${{parameters.defaultPoolName }} + images: + Win11_Azure_Sql: ADO-CI-Win11 + TargetFrameworks: ${{parameters.targetFrameworks }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: ${{parameters.buildPlatforms }} + testSets: ${{parameters.testSets }} + useManagedSNI: ${{parameters.useManagedSNI }} + configSqlFor: azure + operatingSystem: Windows + configProperties: + TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING) + NPConnectionString: $(AZURE_DB_NP_CONN_STRING) + AADAuthorityURL: $(AADAuthorityURL) + ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: + AADPasswordConnectionString: $(AAD_PASSWORD_CONN_STR) + AADServicePrincipalSecret: $(AADServicePrincipalSecret) + AADServicePrincipalId: $(AADServicePrincipalId) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: false + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + # Windows 11 on ARM64 with Azure SQL Server. - windows_azure_arm64_sql: + win11_azure_sql_arm64: pool: ADO-CI-PUBLIC-ARM64-1ES-EUS-POOL images: - Win11_ARM64_Azure_Sql: ADO-WIN11-ARM64 + Win11_Azure_Sql_ARM64: ADO-WIN11-ARM64 TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -537,13 +617,12 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # Linux Ubuntu 20 and 22 with local SQL Server 2022, x64 build platform. - linux_ub20_22_sql_22: + # Linux Ubuntu 24 with local SQL Server 2025. + linux_ub24_sql25: pool: ${{parameters.defaultPoolName }} images: - Ubuntu20_Sql22: ADO-UB20-SQL22 - Ubuntu22_Sql22: ADO-UB22-SQL22 - TargetFrameworks: ${{parameters.targetFrameworksUnix }} + Ubuntu24_Sql25: ADO-UB24-SQL25 + TargetFrameworks: ${{parameters.primaryTargetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] testSets: ${{parameters.testSets }} @@ -561,12 +640,68 @@ stages: LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) AliasName: $(SQLAliasName) - # Linux Ubuntu 22 with Azure SQL Server, x64 build platform. - linux_azure_sql: + # SQL Server 2022 coverage, on Windows and Linux. + # + # The primary configurations run against SQL Server 2025, so these keep + # SQL Server 2022 in the matrix. They are grouped under a single + # conditional so that PR pipelines can opt out of them as a unit. + ${{ if eq(parameters.runSql22Tests, true) }}: + # Windows Server 22 with local SQL Server 2022. + win22_sql22: + pool: ${{parameters.defaultPoolName }} + images: + Win22_Sql22: ADO-MMS22-SQL22 + TargetFrameworks: ${{parameters.targetFrameworks }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: ${{parameters.buildPlatforms }} + testSets: ${{parameters.testSets }} + useManagedSNI: ${{parameters.useManagedSNI }} + configSqlFor: local + operatingSystem: Windows + # config.jsonc properties + configProperties: + TCPConnectionString: $(SQL_TCP_CONN_STRING) + NPConnectionString: $(SQL_NP_CONN_STRING) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: true + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + FileStreamDirectory: $(FileStreamDirectory) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + AliasName: $(SQLAliasName) + SQLRootPath: $(SQL22RootPath) + enableLocalDB: true + + # Linux Ubuntu 22 with local SQL Server 2022. + linux_ub22_sql22: + pool: ${{parameters.defaultPoolName }} + images: + Ubuntu22_Sql22: ADO-UB22-SQL22 + TargetFrameworks: ${{parameters.targetFrameworksUnix }} + netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} + buildPlatforms: [AnyCPU] + testSets: ${{parameters.testSets }} + useManagedSNI: [true] + configSqlFor: local + operatingSystem: Linux + configProperties: + TCPConnectionString: $(SQL_TCP_CONN_STRING) + NPConnectionString: $(SQL_NP_CONN_STRING) + AzureKeyVaultUrl: $(AzureKeyVaultUrl) + AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) + SupportsIntegratedSecurity: false + UserManagedIdentityClientId: $(UserManagedIdentityClientId) + LocalDbAppName: $(LocalDbAppName) + LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) + AliasName: $(SQLAliasName) + + # Linux Ubuntu 24 with Azure SQL Server. + linux_ub24_azure_sql: pool: ${{parameters.defaultPoolName }} images: - Ubuntu22_Azure_Sql: ADO-UB22-SQL22 - TargetFrameworks: ${{parameters.targetFrameworksUnix }} + Ubuntu24_Azure_Sql: ADO-UB24 + TargetFrameworks: ${{parameters.primaryTargetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] testSets: ${{parameters.testSets }} @@ -588,12 +723,11 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # macOS with local SQL Server 2022, x64 build platform. - mac_sql_22: + # macOS with local SQL Server 2025 (docker), x64 build platform. + mac_sql25: pool: Azure Pipelines - hostedPool: true images: - MacOSLatest_Sql22: macos-latest + MacOSLatest_Sql25: macos-latest TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] @@ -614,11 +748,11 @@ stages: # Only run these tests if explicitly enabled, and if we're not a forked repo (which won't # have access to the necessary Library secrets). ${{ if and(eq(parameters.runAlwaysEncryptedTests, true), eq(variables['System.PullRequest.IsFork'], 'False')) }}: - # Windows Server 22 with remote Enclave-enabled SQL Server 2019, x64 build platform. - windows_enclave_sql: + # Windows Server 22 with remote Enclave-enabled SQL Server 2019. + win22_enclave_sql19: pool: ADO-CI-AE-1ES-Pool images: - Win22_Enclave_Sql19: ADO-MMS22-SQL19 + Win22_Enclave_Sql19: ADO-Win25 TargetFrameworks: ${{parameters.targetFrameworks }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: ${{parameters.buildPlatforms }} @@ -642,11 +776,11 @@ stages: LocalDbAppName: $(LocalDbAppName) LocalDbSharedInstanceName: $(LocalDbSharedInstanceName) - # Linux Ubuntu 22 with remote Enclave-enabled SQL Server 2019, x64 build platform. - linux_enclave_sql: + # Linux Ubuntu 24 with remote Enclave-enabled SQL Server 2019. + linux_ub24_enclave_sql19: pool: ADO-CI-AE-1ES-Pool images: - Ubuntu20_Enclave_Sql19: ADO-UB22-Sql22 + Ubuntu24_Enclave_Sql19: ADO-UB24 TargetFrameworks: ${{parameters.targetFrameworksUnix }} netcoreVersionTestUtils: ${{parameters.netcoreVersionTestUtils }} buildPlatforms: [AnyCPU] diff --git a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml index c7fa90860f..a1b5f14216 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-package-reference-pipeline.yml @@ -59,7 +59,9 @@ trigger: branches: include: # GitHub main and release branches. - - main + # + # GOTCHA: Currently disabled on main due to limited resources. + #- main - release/* # ADO main and release branches. @@ -175,6 +177,11 @@ extends: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + # Keep the primary (SQL 2025 / Azure SQL) configurations on the same set of + # target frameworks as everything else. Only the CI-SqlClient pipeline + # opts in to the broader .NET 10.0 coverage on those configurations. + primaryTargetFrameworks: ${{ parameters.targetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} diff --git a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml index 54a1b0e639..ce4db1ce3c 100644 --- a/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml +++ b/eng/pipelines/dotnet-sqlclient-ci-project-reference-pipeline.yml @@ -113,8 +113,9 @@ parameters: # These are _not_ the target frameworks to build the driver packages for. # # Note: We are excluding .NET 10.0 here to avoid consuming too many resources - # during PR pipeline runs, and until we update our 1ES images to include - # Visual Studio 2026 (18.0) whose MSBuild SDK supports .NET 10. + # across every SQL Server image. .NET 10.0 coverage is provided by the + # primaryTargetFrameworks parameter below, which applies to the SQL Server + # 2025 and Azure SQL configurations. # - name: targetFrameworks displayName: Target Frameworks on Windows @@ -126,14 +127,38 @@ parameters: # These are _not_ the target frameworks to build the driver packages for. # # Note: We are excluding .NET 10.0 here to avoid consuming too many resources - # during PR pipeline runs, and until we update our 1ES images to include - # Visual Studio 2026 (18.0) whose MSBuild SDK supports .NET 10. + # across every SQL Server image. .NET 10.0 coverage is provided by the + # primaryTargetFrameworksUnix parameter below, which applies to the SQL + # Server 2025 and Azure SQL configurations. # - name: targetFrameworksUnix displayName: Target Frameworks on Unix type: object default: [net8.0, net9.0] + # The target frameworks used by the primary test configurations (local SQL + # Server 2025 and Azure SQL) on Windows. + # + # net10.0 is included here so that unit, functional, and manual tests get + # .NET 10 coverage without paying for it on every legacy SQL Server image. + # + # Note: The driver itself does not ship a net10.0 target framework, so the + # net10.0 test assemblies resolve the net9.0 driver build. These jobs + # therefore validate the driver running on the .NET 10 runtime, rather than + # a net10.0 build of the driver. + # + - name: primaryTargetFrameworks + displayName: Target Frameworks on Windows (SQL 2025 and Azure SQL) + type: object + default: [net462, net8.0, net9.0, net10.0] + + # The target frameworks used by the primary test configurations (local SQL + # Server 2025 and Azure SQL) on Unix. + - name: primaryTargetFrameworksUnix + displayName: Target Frameworks on Unix (SQL 2025 and Azure SQL) + type: object + default: [net8.0, net9.0, net10.0] + # The timeout, in minutes, for each test job. - name: testJobTimeout displayName: Test job timeout (in minutes) @@ -175,6 +200,8 @@ extends: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + primaryTargetFrameworks: ${{ parameters.primaryTargetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.primaryTargetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} diff --git a/eng/pipelines/github-sync-pipeline.yml b/eng/pipelines/github-sync-pipeline.yml index 699270f4de..eb8c3081c6 100644 --- a/eng/pipelines/github-sync-pipeline.yml +++ b/eng/pipelines/github-sync-pipeline.yml @@ -57,11 +57,16 @@ parameters: type: string default: internal/main +variables: + - template: /eng/pipelines/libraries/ci-build-variables.yml@self + jobs: - job: SyncGitHub displayName: Sync GitHub to ADO pool: - vmImage: 'ubuntu-latest' + name: $(general_purpose_pool_name) + demands: + - imageOverride -equals ADO-UB24 steps: # Check out the ADO repo with full history so we can compare branches. diff --git a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml index 8bf43bd997..3b01faddc5 100644 --- a/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-abstractions-package-ci-job.yml @@ -67,6 +67,18 @@ parameters: # Reference sibling packages as C# projects. - Project + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: pack_abstractions_package_job @@ -75,8 +87,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/pack-azure-package-ci-job.yml b/eng/pipelines/jobs/pack-azure-package-ci-job.yml index 38eaf58fe9..884d4a5c32 100644 --- a/eng/pipelines/jobs/pack-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-azure-package-ci-job.yml @@ -73,6 +73,18 @@ parameters: # Reference sibling packages as C# projects. - Project + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: pack_azure_package_job @@ -81,8 +93,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/pack-logging-package-ci-job.yml b/eng/pipelines/jobs/pack-logging-package-ci-job.yml index 0f0faa1859..1adcf3ac1d 100644 --- a/eng/pipelines/jobs/pack-logging-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-logging-package-ci-job.yml @@ -50,6 +50,18 @@ parameters: - detailed - diagnostic + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: pack_logging_package_job @@ -58,8 +70,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml b/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml index 4f619b3f1b..54938389a5 100644 --- a/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml +++ b/eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml @@ -49,6 +49,18 @@ parameters: - detailed - diagnostic + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + jobs: - job: pack_sqlserver_package_job @@ -57,8 +69,10 @@ jobs: dependsOn: ${{ parameters.dependsOn }} pool: - name: Azure Pipelines - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml index 83366b52a9..c706ca0c57 100644 --- a/eng/pipelines/jobs/test-abstractions-package-ci-job.yml +++ b/eng/pipelines/jobs/test-abstractions-package-ci-job.yml @@ -58,11 +58,15 @@ parameters: default: [] # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string # The pool VM image to use. - - name: vmImage + - name: poolImage type: string jobs: @@ -74,11 +78,11 @@ jobs: # Images provided by Azure Pipelines must be selected using 'vmImage'. ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} + vmImage: ${{ parameters.poolImage }} # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/jobs/test-azure-package-ci-job.yml b/eng/pipelines/jobs/test-azure-package-ci-job.yml index 104401b585..e927eb6e6e 100644 --- a/eng/pipelines/jobs/test-azure-package-ci-job.yml +++ b/eng/pipelines/jobs/test-azure-package-ci-job.yml @@ -98,6 +98,10 @@ parameters: default: [] # The name of the Azure Pipelines pool to use. + # + # NOTE: This value is compared at template-expansion (compile) time to choose between 'vmImage' + # and an imageOverride demand, so the Microsoft-hosted 'Azure Pipelines' pool must be named by + # that exact literal, not by a $(...) macro or $[...] runtime expression. - name: poolName type: string @@ -121,14 +125,8 @@ parameters: type: stepList default: [] - # True if the VM image includes a local SQL Server that supports connections - # via integrated security. - - name: supportsIntegratedSecurity - type: boolean - default: false - # The pool VM image to use. - - name: vmImage + - name: poolImage type: string jobs: @@ -140,11 +138,11 @@ jobs: # Images provided by Azure Pipelines must be selected using 'vmImage'. ${{ if eq(parameters.poolName, 'Azure Pipelines') }}: - vmImage: ${{ parameters.vmImage }} + vmImage: ${{ parameters.poolImage }} # Images provided by 1ES must be selected using a demand. ${{ else }}: demands: - - imageOverride -equals ${{ parameters.vmImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: @@ -252,8 +250,7 @@ jobs: AADServicePrincipalId: $(AADServicePrincipalId) AzureKeyVaultTenantId: $(AzureKeyVaultTenantId) # macOS doesn't support managed identities. - ManagedIdentitySupported: ${{ not(eq(parameters.vmImage, 'macos-latest')) }} - SupportsIntegratedSecurity: ${{ parameters.supportsIntegratedSecurity }} + ManagedIdentitySupported: ${{ not(eq(parameters.poolImage, 'macos-latest')) }} TCPConnectionString: $(AZURE_DB_TCP_CONN_STRING) UserManagedIdentityClientId: $(UserManagedIdentityClientId) WorkloadIdentityFederationServiceConnectionId: $(WorkloadIdentityFederationServiceConnectionId) diff --git a/eng/pipelines/libraries/ci-build-variables.yml b/eng/pipelines/libraries/ci-build-variables.yml index 5486fa9da5..774be86113 100644 --- a/eng/pipelines/libraries/ci-build-variables.yml +++ b/eng/pipelines/libraries/ci-build-variables.yml @@ -9,6 +9,7 @@ # The buildSuffix itself is set by each pipeline via the core template parameter. variables: + - group: sqlclient-pipeline-config-v1 - group: ADO Build properties - group: ADO Test Configuration Properties diff --git a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml index f773ec5e7e..6576627a6a 100644 --- a/eng/pipelines/onebranch/jobs/build-buildproj-job.yml +++ b/eng/pipelines/onebranch/jobs/build-buildproj-job.yml @@ -17,6 +17,12 @@ parameters: - name: apiScanPdbPath type: string + # The APIScan registration version for the package being built. This is the major.minor value + # derived from the canonical package version by the compute_versions stage. The package's + # name/version pair must already be registered with APIScan before the build runs. + - name: apiScanSoftwareVersion + type: string + # True to enable ESRP malware scanning and code signing steps, which should not be run on # non-official pipelines as they access production resources. If true, Signing* parameters must # be provided. @@ -60,8 +66,9 @@ parameters: type: object default: [] - # Positive unsigned 16-bit revision used by build.proj to derive assembly file versions. - - name: revision + # Assembly file version to stamp (required). Pre-computed by the compute-versions stage so no + # build job re-derives it. The assembly version is derived from this by Versions.props. + - name: fileVersion type: string # The full name of the package. This is used in the job name, and to form DLL and PDB filenames @@ -112,13 +119,29 @@ jobs: ob_outputDirectory: '$(JOB_OUTPUT)' - # APIScan per-job configuration for the DLL and PDB folders. + # APIScan per-job configuration. This job template is the single place where the APIScan + # software name and version are set; the pipelines' globalSdl blocks deliberately leave them + # unset so that every scan is attributed to the package it actually covers. ob_sdl_apiscan_softwareFolder: ${{ parameters.apiScanDllPath }} ob_sdl_apiscan_symbolsFolder: ${{ parameters.apiScanPdbPath }} + ob_sdl_apiscan_softwareName: ${{ parameters.packageFullName }} + ob_sdl_apiscan_versionNumber: ${{ parameters.apiScanSoftwareVersion }} + + # SBOM identity for this job's artifact. OneBranch reads the SBOM package name/version only + # from the pipeline's globalSdl block, which has no per-job form, so that block indirects + # through these variables and each build job supplies its own values. Jobs that publish no + # packages set ob_sdl_sbom_enabled to false instead. + sbomPackageName: ${{ parameters.packageFullName }} + sbomPackageVersion: ${{ parameters.packageVersion }} steps: - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self + # Localized resources ship with the SqlClient driver. Validate them before analysis and + # building so missing or untranslated strings fail every SqlClient build. + - ${{ if eq(parameters.packageShortName, 'SqlClient') }}: + - template: /eng/pipelines/onebranch/steps/validate-localization-step.yml@self + - ${{ each package in parameters.dependencies }}: # Build the dependency version arguments passed to the build/pack/analysis steps. The # SqlClient family shares a single version via Central Package Management, and those steps @@ -149,12 +172,14 @@ jobs: # such as _CheckPwshToolRestored that run during RoslynAnalyzers and Build. - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - # Perform Roslyn analysis before building, since this step will clobber build output. + # Run Roslyn analysis. This step is self-contained: it performs its own build into an + # isolated output location, so it can run at any point in the job without clobbering the + # real build output below. - template: /eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml@self parameters: dependencyArguments: $(sqlServerVersionArgument) packageShortName: ${{ parameters.packageShortName }} - revision: ${{ parameters.revision }} + fileVersion: ${{ parameters.fileVersion }} versionPropertySuffix: ${{ parameters.versionPropertySuffix }} packageVersion: ${{ parameters.packageVersion }} @@ -164,7 +189,7 @@ jobs: buildConfiguration: Release dependencyArguments: $(sqlServerVersionArgument) packageShortName: ${{ parameters.packageShortName }} - revision: ${{ parameters.revision }} + fileVersion: ${{ parameters.fileVersion }} versionPropertySuffix: ${{ parameters.versionPropertySuffix }} packageVersion: ${{ parameters.packageVersion }} @@ -212,7 +237,7 @@ jobs: dependencyArguments: $(sqlServerVersionArgument) packageFullName: ${{ parameters.packageFullName }} packageShortName: ${{ parameters.packageShortName }} - revision: ${{ parameters.revision }} + fileVersion: ${{ parameters.fileVersion }} versionPropertySuffix: ${{ parameters.versionPropertySuffix }} packageVersion: ${{ parameters.packageVersion }} diff --git a/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml b/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml index cf7dd6a56b..e63c72bf58 100644 --- a/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml +++ b/eng/pipelines/onebranch/jobs/publish-nuget-package-job.yml @@ -74,6 +74,11 @@ jobs: - name: ob_outputDirectory value: $(JOB_OUTPUT) + # This job republishes an already-built package, whose SBOM came from its build job. It sets + # no sbomPackage* values, so leaving SBOM enabled would emit one with unresolved macros. + - name: ob_sdl_sbom_enabled + value: false + - name: artifactPath value: $(Pipeline.Workspace)/${{ parameters.artifactName }} diff --git a/eng/pipelines/onebranch/jobs/publish-symbols-job.yml b/eng/pipelines/onebranch/jobs/publish-symbols-job.yml index af59e54fc2..ab45b5f4c8 100644 --- a/eng/pipelines/onebranch/jobs/publish-symbols-job.yml +++ b/eng/pipelines/onebranch/jobs/publish-symbols-job.yml @@ -64,6 +64,9 @@ jobs: value: false - name: ob_sdl_binskim_enabled value: false + # No packages are published here, so there is nothing to describe in an SBOM. + - name: ob_sdl_sbom_enabled + value: false # Path where the downloaded artifact will be placed. - name: artifactPath value: '$(Pipeline.Workspace)/${{ parameters.packageFullName }}' diff --git a/eng/pipelines/onebranch/jobs/validate-packages-job.yml b/eng/pipelines/onebranch/jobs/validate-packages-job.yml new file mode 100644 index 0000000000..f1501c7d1d --- /dev/null +++ b/eng/pipelines/onebranch/jobs/validate-packages-job.yml @@ -0,0 +1,206 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Validates every NuGet package produced by this run. +# +# All packages are downloaded into a single tree and validated together in one job, rather than one +# job per package, so that PackageValidator can apply its cross-package rules: the SqlClient family +# must share a single version, and their inter-package dependency ranges must agree. +# +# The job runs on Windows because Authenticode verification has no equivalent on the Linux agents. +# PackageValidator itself is cross-platform, so only the signature checks are OS-bound. + +parameters: + # Package Parameters ----------------------------------------------------- + + - name: abstractionsArtifactsName + type: string + + - name: akvProviderArtifactsName + type: string + + - name: azureArtifactsName + type: string + + - name: loggingArtifactsName + type: string + + - name: sqlClientArtifactsName + type: string + + - name: sqlServerArtifactsName + type: string + + # Version Parameters ----------------------------------------------------- + # Pre-computed by the compute-versions stage. Validation asserts the produced packages carry + # exactly these versions, so nothing here is re-derived. + + - name: sqlClientPackageVersion + type: string + + - name: sqlClientFileVersion + type: string + + - name: sqlServerPackageVersion + type: string + + - name: sqlServerFileVersion + type: string + + # Behaviour Parameters --------------------------------------------------- + + # Whether Microsoft.SqlServer.Server was built this run. When false there is no SqlServer + # artifact to download and no SqlServer package in the drop to validate. + - name: buildSqlServer + type: boolean + + # True for official builds, which sign their packages and assemblies. Signature verification is + # skipped otherwise, because non-official runs deliberately produce unsigned output. + - name: isOfficial + type: boolean + +jobs: + - job: validate_packages + displayName: 'Validate Packages' + + pool: + type: windows + + # 1ES auto-injects Roslyn into any job holding a DotNetCoreCLI build task, which here is only + # the PackageValidator tool build -- never shipped, so out of SDL scope. + templateContext: + sdl: + roslyn: + enabled: false + + variables: + - name: ob_outputDirectory + value: '$(JOB_OUTPUT)' + + # This job inspects already-built packages and produces no assemblies, so it has nothing for + # APIScan or BinSkim to scan and no shipping component to describe in an SBOM. The build + # jobs cover all three for the packages they produce. + - name: ob_sdl_apiscan_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_sbom_enabled + value: false + + # Every package artifact is downloaded beneath this root, each into its own subdirectory so + # that identically-named files from different packages cannot collide. + - name: packagesRoot + value: '$(Pipeline.Workspace)/validate-packages' + + - name: extractRoot + value: '$(Pipeline.Workspace)/validate-extract' + + steps: + - template: /eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self + + # Only the packages themselves are needed, not the full build output each artifact carries. + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Logging' + inputs: + artifactName: '${{ parameters.loggingArtifactsName }}' + targetPath: '$(packagesRoot)/Logging' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Abstractions' + inputs: + artifactName: '${{ parameters.abstractionsArtifactsName }}' + targetPath: '$(packagesRoot)/Abstractions' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - SqlClient' + inputs: + artifactName: '${{ parameters.sqlClientArtifactsName }}' + targetPath: '$(packagesRoot)/SqlClient' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - Azure' + inputs: + artifactName: '${{ parameters.azureArtifactsName }}' + targetPath: '$(packagesRoot)/Azure' + patterns: '**/*.*nupkg' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - AkvProvider' + inputs: + artifactName: '${{ parameters.akvProviderArtifactsName }}' + targetPath: '$(packagesRoot)/AkvProvider' + patterns: '**/*.*nupkg' + + - ${{ if eq(parameters.buildSqlServer, true) }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Packages - SqlServer' + inputs: + artifactName: '${{ parameters.sqlServerArtifactsName }}' + targetPath: '$(packagesRoot)/SqlServer' + patterns: '**/*.*nupkg' + + # PackageValidator targets net10.0, which the repo's global.json already pins. + - template: /eng/pipelines/common/steps/install-dotnet.yml@self + + - template: /eng/pipelines/onebranch/steps/validate-packages-step.yml@self + parameters: + packagesPath: '$(packagesRoot)' + reportPath: '$(JOB_OUTPUT)/validation/package-validation.json' + sqlClientPackageVersion: '${{ parameters.sqlClientPackageVersion }}' + sqlClientFileVersion: '${{ parameters.sqlClientFileVersion }}' + # Omitted when SqlServer is not built: its package is absent from the drop, and the + # validator rejects an expectation with an empty value. + ${{ if eq(parameters.buildSqlServer, true) }}: + sqlServerPackageVersion: '${{ parameters.sqlServerPackageVersion }}' + sqlServerFileVersion: '${{ parameters.sqlServerFileVersion }}' + # The error severity covers only error-severity findings, so every warning/info category + # this job relies on must be named explicitly: missing-symbols, dependency-inconsistency + # and delay-signed are warnings, and unsigned and package-unsigned are info. + # + # Strong-name signing is unconditional in build-buildproj-step.yml, so delay-signed and + # unsigned gate on every run. NuGet package signing is ESRP and runs on official builds + # only, so package-unsigned would fire on every non-official build. + ${{ if eq(parameters.isOfficial, true) }}: + failOn: + - error + - missing-symbols + - dependency-inconsistency + - delay-signed + - unsigned + - package-unsigned + ${{ else }}: + failOn: + - error + - missing-symbols + - dependency-inconsistency + - delay-signed + - unsigned + + # Signature verification, official builds only. PackageValidator reports strong-name and + # NuGet signature *presence* cross-platform; these steps additionally verify that the + # signatures are trusted, which requires the Windows trust store. + - ${{ if eq(parameters.isOfficial, true) }}: + - task: PowerShell@2 + displayName: 'Verify NuGet package signatures' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 + arguments: >- + -PackagesPath "$(packagesRoot)" + + - task: PowerShell@2 + displayName: 'Verify assembly Authenticode signatures' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 + arguments: >- + -PackagesPath "$(packagesRoot)" + -ExtractPath "$(extractRoot)" diff --git a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml b/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml deleted file mode 100644 index afa5aa5918..0000000000 --- a/eng/pipelines/onebranch/jobs/validate-signed-package-job.yml +++ /dev/null @@ -1,386 +0,0 @@ -################################################################################# -# Licensed to the .NET Foundation under one or more agreements. # -# The .NET Foundation licenses this file to you under the MIT license. # -# See the LICENSE file in the project root for more information. # -################################################################################# -parameters: - # The name of the pipeline artifact to download that contains the SqlClient NuGet package. - - name: artifactName - type: string - - # List of versions of dotnet that are *allowed to exist* in the NuGet package. Separators do not - # matter as the folders in lib, runtime, etc are simply checked to see if they exist in this - # string. - - name: expectedDotnetVersions - type: string - default: 'net462;net8.0;net9.0;netstandard2.0' - - # Expected file version of the assemblies within the package. This should be of the form: - # (major).(minor).(patch).(buildNumber) - - name: expectedFileVersion - type: string - - # List of folders that are *allowed to exist* in the NuGet package. Separators do not matter as - # the folders are simply checked to see if they exist in this string. - - name: expectedFolderNames - type: string - default: 'lib;ref;runtimes' - - # Expected NuGet package version. Used to build the installation path. This should be of the - # form: (major).(minor).(patch)[-preview(preview_number)] - - name: expectedPackageVersion - type: string - - # True if this build is an official build. This will be used to gate some checks - # that only apply to official builds, such as signature verification. - - name: isOfficial - type: boolean - -jobs: - - job: validate_nuget_package - displayName: "Validate NuGet package" - - pool: - type: windows # read more about custom job pool types at https://aka.ms/obpipelines/yaml/jobs - isCustom: true - name: ADO-1ES-Pool - vmImage: "ADO-MMS22-SQL19" - - variables: # More settings at https://aka.ms/obpipelines/yaml/jobs - - # Path within the downloaded artifact where NuGet packages are located. - - name: artifactPath - value: '$(Pipeline.Workspace)\${{ parameters.artifactName }}' - - # Path to the SqlClient NuGet package after installation. This path will only exist once the package - # been installed. - - name: nugetPackageInstallPath - value: '$(Pipeline.Workspace)\nugetPackageInstalls\Microsoft.Data.SqlClient.${{ parameters.expectedPackageVersion }}' - - # Root folder where NuGet package will be installed locally - - name: nugetPackageInstallRoot - value: '$(Pipeline.Workspace)\nugetPackageInstalls' - - steps: - - template: '/eng/pipelines/onebranch/steps/script-output-environment-variables-step.yml@self' - - - task: NuGetToolInstaller@1 - displayName: "Install NuGet" - - - powershell: | - echo "> 1. List all local cache directory paths" - nuget locals all -List - - echo "> 2. Clear all files from local cache directories" - nuget locals all -Clear - displayName: "Clear local cache" - - # Download NuGet packages from the specified build artifact. - - download: current - artifact: ${{ parameters.artifactName }} - patterns: "**/*.*nupkg" - displayName: "Download NuGet Package" - - # Verify secure signatures on the NuGet packages. - # NOTE: Packages will only be signed if the build is official. - - ${{ if eq(parameters.isOfficial, true) }}: - - powershell: | - # Propagate parameters to PS variables ####################### - $artifactPath = "${{ variables.artifactPath }}" - echo "artifactPath= $artifactPath" - - # Discover packages ########################################## - $packageFiles = Get-ChildItem -Path $artifactPath -Recurse -File -Include *.nupkg,*.snupkg - if ($packageFiles.Count -eq 0) - { - Write-Error "No NuGet package files were found under '$artifactPath'." - } - - # Verify package signatures ################################## - echo "> 1. Verify signature of source package(s)" - $packageFiles | Where-Object { $_.Extension -eq ".nupkg" } | ForEach-Object { - nuget verify -All $_.FullName - } - - echo "> 2. Verify signature of symbols package(s)" - $packageFiles | Where-Object { $_.Extension -eq ".snupkg" } | ForEach-Object { - nuget verify -All $_.FullName - } - displayName: "Verify nuget signature" - - # Install NuGet package to the temporary directory - - powershell: | - # Propagate pipeline to PS variables ############################# - $artifactPath = "${{ variables.artifactPath }}" - echo "artifactPath= $artifactPath" - - $expectedPackageVersion = "${{ parameters.expectedPackageVersion }}" - echo "expectedPackageVersion= $expectedPackageVersion" - - $nugetPackageInstallRoot = "${{ variables.nugetPackageInstallRoot }}" - echo "nugetPackageInstallRoot= $nugetPackageInstallRoot" - - # Find the SqlClient NuGet package ############################### - Get-ChildItem "$artifactPath" -Recurse - - $packagePaths = @(Get-ChildItem -Path $artifactPath -Recurse -File -Filter "Microsoft.Data.SqlClient.$expectedPackageVersion.nupkg") - if ($packagePaths.Count -eq 0) - { - Write-Error "Unable to find Microsoft.Data.SqlClient.$expectedPackageVersion.nupkg under '$artifactPath'." - } - if ($packagePaths.Count -gt 1) - { - Write-Error "Multiple Microsoft.Data.SqlClient.$expectedPackageVersion.nupkg files were found under '$artifactPath'." - } - - $packageSource = Split-Path -Path $packagePaths[0].FullName -Parent - echo "Found package path: $($packagePaths[0].FullName)" - echo "Using package source: $packageSource" - - # Install NuGet Package ########################################## - echo "> 1. Installing Microsoft.Data.SqlClient NuGet package..." - Install-Package ` - -Name "Microsoft.Data.SqlClient" ` - -Source "$packageSource" ` - -Destination $nugetPackageInstallRoot ` - -Force ` - -SkipDependencies - - echo "> 2. Listing contents of installed Microsoft.Data.SqlClient NuGet package:" - Write-Host $nugetPackageInstallRoot - Get-ChildItem $nugetPackageInstallRoot - displayName: "Install NuGet Package" - - # Find all DLL files in the installed NuGet package, verify each is signed with a strong name - - powershell: | - # Propagate pipeline to PS variables ############################# - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify strong name signing ##################################### - echo "> 1. Verifying strong name signing of DLLs ..." - - # @TODO: This path seems brittle to VS upgrades, can we make it more flexible? - $snPath = "C:\Program Files (x86)\Microsoft SDKs\Windows\*\bin\NETFX 4.8.1 Tools\sn.exe" - - $dllFiles = Get-ChildItem -Path $nugetPackageInstallPath -Recurse -Filter *.dll - $badDlls = @() - foreach ($file in $dllFiles) - { - # Run sn.exe to verify the strong name on each dll - $result = & $snPath -vf $file.FullName - Write-OutPut $result - - # if the dll is not valid, it would be delay signed or test-signed which is not meant for production - if($result[$result.Length-1] -notlike "* is valid") - { - $badDlls += $result[$result.Length-1] - } - } - if($badDlls.Count -gt 0) - { - Write-OutPut "Error: Invalid dlls are detected. Check the list below:" - foreach($dll in $badDlls) - { - Write-Output $dll - } - Exit -1 - } - Write-Host "Strong name has been verified for all dlls" - displayName: "Verify assembly strong names" - - # Validate that the folders in the nuget are expected - # @TODO: This does not verify we are not missing any folders, only that the folders that - # exist are expected to exist. - - powershell: | - # Propagate pipeline to PS variables ############################# - $expectedFolderNames = "${{ parameters.expectedFolderNames }}" - echo "expectedFolderNames= $expectedFolderNames" - - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify folders are expected #################################### - Get-ChildItem -Path $nugetPackageInstallPath -Directory | select Name | foreach { - if($expectedFolderNames.contains($_.Name)){ - Write-Host expected folder name verfied: $_.Name - } - } - displayName: "Verify NuGet Root Folder Structure" - - # Validate that the folders within the root folders of the nuget are expected - # @TODO: This does not verify we are not missing any folders, only that the folders that - # exist are expected to exist. - - powershell: | - # Propagate pipeline to PS variables ############################# - $expectedDotnetVersions = "${{ parameters.expectedDotnetVersions }}" - echo "expectedDotnetVersions= $expectedDotnetVersions" - - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify folders are expected #################################### - # Checks the version of DotNetFramework and DotNet - $countErr = 0 - $countPass = 0 - $excludNamesFromRuntimeFolder = 'lib','win','unix' - - Get-ChildItem -Path $nugetPackageInstallPath -Directory | foreach { - $parentname=$_.Name - Write-Host $_.FullName -ForegroundColor yellow - - if($_.Name -ne 'runtimes') { - Get-ChildItem -Path $_.FullName -Directory | select Name | foreach { - if($expectedDotnetVersions.Contains($_.Name)){ - Write-Host "`tExpected version verified in $parentname": $_.Name -ForegroundColor green - $countPass += 1 - } - else{ - Write-Host "`tUnexpected version detected in $parentname": $_.Name - $countErr += 1 - } - } - } - - elseif ($_.Name -eq 'runtimes'){ - Get-ChildItem -Depth 3 -Path $_.FullName -Exclude $excludNamesFromRuntimeFolder -Directory | foreach{ - if('${{ parameters.expectedDotnetVersions }}'.Contains($_.Name)){ - Write-Host "`tExpected version verfied in $parentname": $_.Name - $countPass += 1 - } - else{ - Write-Host "`tUnexpected version detected": $_.Name -ForegroundColor Red - $countErr += 1 - } - } - } - else{ - Write-Host "`tUnknown folder " $_.Name -ForegroundColor Red - Exit -1 - } - } - - Write-Host "_______________" - Write-Host "Expected: $countPass" - Write-Host "Unexpected: $countErr" - Write-Host "_______________" - if ($countErr -ne 0) - { - Write-Host "Unexpected versions are detected!" -ForegroundColor Red - Exit -1 - } - displayName: "Verify NuGet DotNet Versions " - - - powershell: | - # Propagate pipeline to PS variables ############################# - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify DLL Hierarchy ########################################### - foreach( $folderName in (Get-ChildItem -Path $nugetPackageInstallPath -Directory).Name) - { - # List all Childerns of the Path - Get-ChildItem -Path $nugetPackageInstallPath\$folderName -Recurse -File - $subFiles = Get-ChildItem -Path $nugetPackageInstallPath\$folderName -Recurse -File - - foreach($file in $subFiles) - { - if($subFiles[0].Name -like "*.dll" ) - { - Write-Host $subFiles[0].Name -ForegroundColor Green - Write-Host $subFiles[1].Name -ForegroundColor Green - if(($folderName -eq 'lib') -or ($folderName -eq 'ref')) - { - if($subFiles[2].Name -like "*.dll") - { - Write-Host $subFiles[2].Name -ForegroundColor Green - } - else - { - $subFiles[2].Name - Write-Host "Expected file pattern for localization did not match to *.dll" -ForegroundColor Red - Exit -1 - } - } - } - else - { - $subFiles[0].Name - $subFiles[1].Name - Write-Host "Expected file pattern did not match to *.dll" -ForegroundColor Red - Exit -1 - } - } - } - displayName: 'Verify all DLLs unzipped match "expected" hierarchy' - - # Verify that all DLLs are authenticode signed - # NOTE: This signing is only performed on official builds. - - ${{ if eq(parameters.isOfficial, true) }}: - - powershell: | - # Propagate pipeline to PS variables ############################# - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Verify authenticode signature of DLLs ########################## - $dlls = Get-ChildItem -Path $nugetPackageInstallPath -Recurse -Include *.dll - foreach ($status in $dlls | Get-AuthenticodeSignature) - { - if ($status.Status -eq "Valid") - { - Write-Host $status.Status $status.Path - } - else - { - Write-Host "dll status of '$status.Path' is not valid!" -ForegroundColor Red - $status - Exit -1 - } - } - displayName: "Verify all dlls status are Valid" - - - powershell: | - # Propagate pipeline to PS variables ############################# - $expectedFileVersion = "${{ parameters.expectedFileVersion }}" - echo "expectedFileVersion= $expectedFileVersion" - - $expectedPackageVersion = "${{ parameters.expectedPackageVersion }}" - echo "expectedPackageVersion= $expectedPackageVersion" - - $nugetPackageInstallPath = "${{ variables.nugetPackageInstallPath }}" - echo "nugetPackageInstallPath= $nugetPackageInstallPath" - - # Validate ProductVersion and FileVersion fields ################# - $failed = 0 - foreach ( $pVersion in Get-ChildItem *.dll -Path $nugetPackageInstallPath -Recurse | ForEach-Object versioninfo ) - { - if ($pVersion.ProductVersion -Like $expectedPackageVersion + '*') - { - Write-Host -ForegroundColor Green "Correct ProductVersion detected for $($pVersion.FileName): $($pVersion.ProductVersion)" - } - else - { - Write-Host -ForegroundColor Red "Wrong ProductVersion detected for $($pVersion.FileName); expected: $expectedPackageVersion; found: $($pVersion.ProductVersion)" - $failed = 1 - } - - if ($pVersion.FileVersion -eq $expectedFileVersion) - { - Write-Host -ForegroundColor Green "Correct FileVersion detected for $($pVersion.FileName): $($pVersion.FileVersion)" - } - else - { - Write-Host -ForegroundColor Red "Wrong FileVersion detected for $($pVersion.FileName); expected $expectedFileVersion; found: $($pVersion.FileVersion)" - $failed = 1 - } - - # @TODO: We should do a check for assembly version here. - } - - if ($failed -ne 0) - { - Exit -1 - } - - Get-ChildItem *.dll -Path $nugetPackageInstallPath -Recurse | ForEach-Object VersionInfo | Format-List - displayName: 'Verify "File Version" matches expected values for DLLs' diff --git a/eng/pipelines/onebranch/scripts/compute-versions.ps1 b/eng/pipelines/onebranch/scripts/compute-versions.ps1 index ca3eb2c9d9..4bce0b1495 100644 --- a/eng/pipelines/onebranch/scripts/compute-versions.ps1 +++ b/eng/pipelines/onebranch/scripts/compute-versions.ps1 @@ -12,61 +12,39 @@ The published SqlServer version is needed when SqlServer is not built because downstream SqlClient projects restore that existing package from NuGet. - Two mutually exclusive package version shapes are supported: - - When AddRevision is true, BuildNumber is ignored if specified. For non-preview releases, versions - will be 1.2.3.. For preview releases, versions will be 1.2.3.-previewX. - - When AddRevision is false, BuildNumber must be specified. Non-preview versions will be 1.2.3, and - preview releases will be 1.2.3-previewX.. - - This asymmetry maintains the existing package versioning practice where previews include a - date-coded build number, but normal releases do not. + Package versions take a single shape, produced by Versions.props from the pipeline build number. + Preview versions carry the full build number after the prerelease suffix, such as + 1.2.3-preview1.26238.3. Stable versions are left exactly as declared in Versions.props, such as + 1.2.3, because released packages are not stamped with a build number. File versions are always four-part and always carry a build number in the fourth component, even - when the package version does not: - - - AddRevision true. The fourth component is the mapped revision, so package and file versions - agree. A package version of 1.2.3.34430-preview1 has file version 1.2.3.34430. + when the package version does not. Versions.props derives that component from the date segment of + BuildNumber, so a package version of 1.2.3-preview1.26238.3 has file version 1.2.3.26238, and a + stable package version of 1.2.3 still has file version 1.2.3.26238. That segment is date-coded, + so repeated runs on the same day share a file version even though their package versions differ. - - AddRevision false. The fourth component is the first segment of BuildNumber. A package - version of 1.2.3-preview1.26238.3 has file version 1.2.3.26238, and a stable package version - of 1.2.3 still has file version 1.2.3.26238. This keeps every file version date-encoded and - traceable back to the run that produced it. - - The supplied revision is mapped to the range 1 through 65535 before canonical versions - are evaluated so every file version has a valid fourth component. Revisions above 65535 wrap - through the valid unsigned 16-bit file-version range. When AddRevision is true the script logs - the mapping because the revision may then collide with an earlier run. An unbuilt SqlServer - package is never revised or stamped with a build number because its effective version must - continue to identify the package that already exists on NuGet. + An unbuilt SqlServer package is never stamped with a build number because its effective version + must continue to identify the package that already exists on NuGet. The script emits these output variables for downstream stages: - - VersionRevision - SqlClientPackageVersion + - SqlClientFileVersion - SqlServerPackageVersion + - SqlServerFileVersion + - SqlClientApiScanVersion + - SqlServerApiScanVersion .PARAMETER ProjectPath Absolute or relative path to the repository build.proj file. -.PARAMETER Revision - Positive integer used to distinguish versions. Values above 65535 are wrapped into the unsigned - 16-bit revision range. Only consumed when AddRevision is true. - .PARAMETER BuildNumber - Pipeline build number in the form ., such as 26238.3. Required when AddRevision is - false, and ignored when AddRevision is true. When required, it is appended to prerelease package - versions and its first segment becomes the file-version build number, so file versions remain - date-encoded. + Pipeline build number in the form ., such as 26238.3. Versions.props appends it to + prerelease package versions and derives the file-version build number from its date segment. .PARAMETER BuildSqlServer Whether this run builds Microsoft.SqlServer.Server. When false, the effective SqlServer package version is its last published version and its file version is not consumed downstream. -.PARAMETER AddRevision - Whether to insert the revision into package versions built during this run. - Defaults to false in both top-level OneBranch pipelines. - .PARAMETER DotnetPath dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter primarily supports isolated testing and specialized agent configurations. @@ -74,33 +52,19 @@ .EXAMPLE ./compute-versions.ps1 ` -ProjectPath ./build.proj ` - -Revision 165500 ` -BuildNumber 26238.3 ` - -BuildSqlServer $true ` - -AddRevision $false + -BuildSqlServer $true - Computes versions for an official-style run that builds SqlServer. Prerelease package versions - become 7.1.0-preview3.26238.3 and file versions become 7.1.0.26238. + Computes versions for a run that builds SqlServer. Prerelease package versions become + 7.1.0-preview3.26238.3 and file versions become 7.1.0.26238. .EXAMPLE ./compute-versions.ps1 ` -ProjectPath ./build.proj ` - -Revision 165500 ` - -BuildSqlServer $true ` - -AddRevision $true - - Computes versions for a run that builds SqlServer and appends mapped revision 34430 to both - package families and their file versions. BuildNumber is not supplied because AddRevision is - true. - -.EXAMPLE - ./compute-versions.ps1 ` - -ProjectPath ./build.proj ` - -Revision 165500 ` - -BuildSqlServer $false ` - -AddRevision $true + -BuildNumber 26238.3 ` + -BuildSqlServer $false - Revises the SqlClient family versions while retaining SqlServerPublishedVersion for dependency + Stamps the SqlClient family versions while retaining SqlServerPublishedVersion for dependency restore because SqlServer is not built in this run. .NOTES @@ -115,20 +79,13 @@ param( [ValidateScript({ Test-Path -LiteralPath $_ -PathType Leaf })] [string]$ProjectPath, - [Parameter(Mandatory = $true, HelpMessage = "Positive integer version revision.")] - [ValidateRange(1, [long]::MaxValue)] - [long]$Revision, - - [Parameter(HelpMessage = "Pipeline build number, such as 26238.3. Required when AddRevision is false.")] - [ValidatePattern("^$|^\d+\.\d+$")] - [string]$BuildNumber = "", + [Parameter(Mandatory = $true, HelpMessage = "Pipeline build number, such as 26238.3.")] + [ValidatePattern("^\d+\.\d+$")] + [string]$BuildNumber, [Parameter(Mandatory = $true, HelpMessage = "Whether Microsoft.SqlServer.Server is built in this run.")] [bool]$BuildSqlServer, - [Parameter(Mandatory = $true, HelpMessage = "Whether to append the revision to built package versions.")] - [bool]$AddRevision, - [Parameter(HelpMessage = "dotnet executable to invoke.")] [ValidateNotNullOrEmpty()] [string]$DotnetPath = "dotnet" @@ -137,11 +94,6 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" -$wrappedRevision = (($Revision - 1) % 65535) + 1 -if ($AddRevision -and $Revision -gt 65535) { - Write-Host "Revision $Revision exceeds the unsigned 16-bit limit and wrapped to $wrappedRevision; this revision may collide with an earlier run." -} - <# .SYNOPSIS Extracts the first value associated with a labeled GetVersions target output line. @@ -177,7 +129,7 @@ function Get-LabeledValue { GetVersions target suffix: SqlClient or SqlServer. .OUTPUTS - An object containing PackageVersion and PublishedVersion. + An object containing PackageVersion, FileVersion, and PublishedVersion. #> function Get-CanonicalVersions { param( @@ -186,91 +138,48 @@ function Get-CanonicalVersions { ) $output = & $DotnetPath build $ProjectPath ` - -t:"GetVersions${Label}" -v:m -nologo -p:BuildNumber=$wrappedRevision 2>&1 + -t:"GetVersions${Label}" -v:m -nologo -p:BuildNumber=$BuildNumber 2>&1 if ($LASTEXITCODE -ne 0) { throw ($output -join [Environment]::NewLine) } $packageVersion = Get-LabeledValue -Output $output -Label "PackageVersion" + $fileVersion = Get-LabeledValue -Output $output -Label "FileVersion" $publishedVersion = Get-LabeledValue -Output $output -Label "PublishedVersion" if ([string]::IsNullOrWhiteSpace($packageVersion)) { throw "Failed to extract PackageVersion for ${Label}.`n$($output -join [Environment]::NewLine)" } + if ([string]::IsNullOrWhiteSpace($fileVersion)) { + throw "Failed to extract FileVersion for ${Label}.`n$($output -join [Environment]::NewLine)" + } [pscustomobject]@{ PackageVersion = $packageVersion + FileVersion = $fileVersion PublishedVersion = $publishedVersion } } <# .SYNOPSIS - Inserts a numeric revision before a package version's prerelease suffix. + Extracts the major.minor components from a package version. .PARAMETER Version - Package version with a three-part numeric base and optional prerelease suffix. - -.PARAMETER Revision - Revision in the unsigned 16-bit file-version range. + Package version beginning with a numeric major.minor pair. .OUTPUTS - A four-part package version preserving the original prerelease suffix. + The major.minor version pair. #> -function Add-VersionRevision { +function Get-MajorMinorVersion { param( - [string]$Version, - [ValidateRange(1, 65535)] - [int]$Revision + [string]$Version ) - $parts = $Version -split "-", 2 - if ($parts[0] -notmatch "^\d+\.\d+\.\d+$") { - throw "Expected a three-part numeric version base, but received '$Version'." - } - - $versionWithRevision = "$($parts[0]).$Revision" - if ($parts.Count -eq 2) { - return "$versionWithRevision-$($parts[1])" + if ($Version -notmatch "^(\d+)\.(\d+)(?:\.|-|$)") { + throw "Unable to derive a major.minor version from package version '$Version'." } - return $versionWithRevision -} - -<# -.SYNOPSIS - Appends a pipeline build number to a prerelease package version. - -.DESCRIPTION - Reproduces the version shape used by earlier previews, where the build number follows the - prerelease suffix, such as 7.1.0-preview3.26238.3. Stable versions are returned unchanged - because released packages are not stamped with a build number. - -.PARAMETER Version - Package version with a three-part numeric base and optional prerelease suffix. - -.PARAMETER BuildNumber - Pipeline build number to append. - -.OUTPUTS - The package version with the build number appended to its prerelease suffix, or the original - version when it carries no prerelease suffix. -#> -function Add-VersionBuildNumber { - param( - [string]$Version, - [string]$BuildNumber - ) - - $parts = $Version -split "-", 2 - if ($parts[0] -notmatch "^\d+\.\d+\.\d+$") { - throw "Expected a three-part numeric version base, but received '$Version'." - } - - if ($parts.Count -eq 2) { - return "$($parts[0])-$($parts[1]).$BuildNumber" - } - - return $Version + return "$($Matches[1]).$($Matches[2])" } <# @@ -292,14 +201,18 @@ function Set-PipelineOutputVariable { Write-Host "##vso[task.setvariable variable=${Name};isOutput=true]$Value" } -Write-Host "Extracting versions with revision=$Revision (wrapped=$wrappedRevision)..." +Write-Host "Extracting versions with build number $BuildNumber..." $sqlClientVersions = Get-CanonicalVersions -Label "SqlClient" $sqlServerVersions = Get-CanonicalVersions -Label "SqlServer" -Write-Host " SqlClient: pkg=$($sqlClientVersions.PackageVersion)" -Write-Host " SqlServer: pkg=$($sqlServerVersions.PackageVersion) pub=$($sqlServerVersions.PublishedVersion)" +Write-Host " SqlClient: pkg=$($sqlClientVersions.PackageVersion) file=$($sqlClientVersions.FileVersion)" +Write-Host " SqlServer: pkg=$($sqlServerVersions.PackageVersion) file=$($sqlServerVersions.FileVersion) pub=$($sqlServerVersions.PublishedVersion)" $sqlClientPackageVersion = $sqlClientVersions.PackageVersion +$sqlClientFileVersion = $sqlClientVersions.FileVersion + +# An unbuilt SqlServer resolves to its published version, which no build job stamps, so it has no +# effective file version. $sqlServerPackageVersion = if ($BuildSqlServer) { $sqlServerVersions.PackageVersion } else { @@ -308,47 +221,22 @@ $sqlServerPackageVersion = if ($BuildSqlServer) { } $sqlServerVersions.PublishedVersion } +$sqlServerFileVersion = if ($BuildSqlServer) { $sqlServerVersions.FileVersion } else { "" } -$fileVersionBuildNumber = $null - -if ($AddRevision) { - $sqlClientPackageVersion = Add-VersionRevision ` - -Version $sqlClientPackageVersion ` - -Revision $wrappedRevision - if ($BuildSqlServer) { - $sqlServerPackageVersion = Add-VersionRevision ` - -Version $sqlServerPackageVersion ` - -Revision $wrappedRevision - } - - # The revision is already the fourth component of the package version, so it is also the - # file-version build number. - $fileVersionBuildNumber = $wrappedRevision - Write-Host "Version revision: $wrappedRevision (input=$Revision)" -} -else { - if ([string]::IsNullOrWhiteSpace($BuildNumber)) { - throw "BuildNumber is required when AddRevision is false." - } - - $fileVersionBuildNumber = $BuildNumber.Split(".")[0] - - $sqlClientPackageVersion = Add-VersionBuildNumber ` - -Version $sqlClientPackageVersion ` - -BuildNumber $BuildNumber - if ($BuildSqlServer) { - $sqlServerPackageVersion = Add-VersionBuildNumber ` - -Version $sqlServerPackageVersion ` - -BuildNumber $BuildNumber - } +Write-Host "Effective versions:" +Write-Host " SqlClient (family): $sqlClientPackageVersion (file $sqlClientFileVersion)" +Write-Host " SqlServer: $sqlServerPackageVersion (file $sqlServerFileVersion)" - Write-Host "Version build number: $BuildNumber (file version build number=$fileVersionBuildNumber)" -} +$sqlClientApiScanVersion = Get-MajorMinorVersion -Version $sqlClientPackageVersion +$sqlServerApiScanVersion = Get-MajorMinorVersion -Version $sqlServerPackageVersion -Write-Host "Effective versions:" -Write-Host " SqlClient (family): $sqlClientPackageVersion" -Write-Host " SqlServer: $sqlServerPackageVersion" +Write-Host "APIScan registration versions:" +Write-Host " SqlClient (family): $sqlClientApiScanVersion" +Write-Host " SqlServer: $sqlServerApiScanVersion" Set-PipelineOutputVariable -Name "SqlClientPackageVersion" -Value $sqlClientPackageVersion +Set-PipelineOutputVariable -Name "SqlClientFileVersion" -Value $sqlClientFileVersion Set-PipelineOutputVariable -Name "SqlServerPackageVersion" -Value $sqlServerPackageVersion -Set-PipelineOutputVariable -Name "VersionRevision" -Value $fileVersionBuildNumber +Set-PipelineOutputVariable -Name "SqlServerFileVersion" -Value $sqlServerFileVersion +Set-PipelineOutputVariable -Name "SqlClientApiScanVersion" -Value $sqlClientApiScanVersion +Set-PipelineOutputVariable -Name "SqlServerApiScanVersion" -Value $sqlServerApiScanVersion diff --git a/eng/pipelines/onebranch/scripts/tests/README.md b/eng/pipelines/onebranch/scripts/tests/README.md index bda88d58a1..593c7768d6 100644 --- a/eng/pipelines/onebranch/scripts/tests/README.md +++ b/eng/pipelines/onebranch/scripts/tests/README.md @@ -31,15 +31,23 @@ Invoke-Pester ./publish-symbols.Tests.ps1 -Output Detailed | Area | What's tested | | --------------------- | ---------------------------------------------------------------- | -| Version computation | Canonical output parsing, revisions, wrapping, effective package selection, and failures | +| Version computation | Canonical output parsing, effective package selection, target version composition, and failures | +| Localization validation | Missing, obsolete, or empty strings, English-value matches, and culture-specific allowlisting | | Parameter validation | Empty strings rejected for all mandatory parameters | | URL construction | Base URL, register URL, request URL built from parameters | | Request bodies | Registration body, default publish flags, flag overrides | | Error handling | Token failure, registration failure, publish failure, status failure — all verify expanded URI in error message | | Status validation | Detects Failed/Cancelled results, respects PublishToInternal/PublishToPublic flags, passes on Succeeded/Pending | +| Package validation | Wildcard vs per-id version expectations, SqlServer omitted when unbuilt, gate tokens, report written before gating, exit-code handling | +| Package signatures | Every package and symbol package verified, all failures reported before throwing | +| Assembly signatures | Package expansion, native binaries under `runtimes/` included, stale expansions replaced, all unsigned assemblies reported | ## Notes - All external calls (`az`, `Invoke-RestMethod`) are mocked — no network access or Azure credentials are required. -- Version tests mock `dotnet`, so they do not invoke MSBuild or require a restored repository. +- Script-level version tests mock `dotnet`; package-composition tests invoke the real MSBuild + `GetVersionsSqlClient` and `GetVersionsSqlServer` targets. +- `Get-AuthenticodeSignature` is Windows-only, so the assembly-signature tests declare a stub when + it is absent. Only the signature lookup is substituted; package expansion and reporting run for + real against packages built in the test's temporary directory. - Tests validate scripts in the parent directory relative to this directory. diff --git a/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 index ba109c4b71..7a9e9779cb 100644 --- a/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 +++ b/eng/pipelines/onebranch/scripts/tests/compute-versions.Tests.ps1 @@ -4,7 +4,9 @@ #> BeforeAll { + $script:repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..' '..' '..' '..' '..') $scriptPath = Join-Path $PSScriptRoot '..' 'compute-versions.ps1' + $buildProjectPath = Resolve-Path (Join-Path $script:repoRoot 'build.proj') $projectPath = Join-Path $TestDrive 'build.proj' Set-Content -LiteralPath $projectPath -Value '' @@ -19,26 +21,23 @@ BeforeAll { function Invoke-ComputeVersions { param( - [long]$Revision = 42, - [string]$BuildNumber = '', - [bool]$BuildSqlServer = $true, - [bool]$AddRevision = $true + [string]$BuildNumber = $script:testBuildNumber, + [bool]$BuildSqlServer = $true ) & $scriptPath ` -ProjectPath $projectPath ` - -Revision $Revision ` -BuildNumber $BuildNumber ` - -BuildSqlServer $BuildSqlServer ` - -AddRevision $AddRevision *>&1 | Out-String + -BuildSqlServer $BuildSqlServer *>&1 | Out-String } # Alternates between the SqlClient and SqlServer GetVersions targets, which the script always - # invokes in that order. + # invokes in that order. The versions returned here are already stamped, because Versions.props + # applies the build number before the script ever sees them. function Set-DotnetMock { param( - [string]$SqlClientPackageVersion = '7.1.0-preview3', - [string]$SqlServerPackageVersion = '1.1.0-preview1' + [string]$SqlClientPackageVersion = "7.1.0-preview3.$script:testBuildNumber", + [string]$SqlServerPackageVersion = "1.1.0-preview1.$script:testBuildNumber" ) $global:computeVersionsDotnetCallCount = 0 @@ -48,12 +47,14 @@ BeforeAll { if ($global:computeVersionsDotnetCallCount % 2 -eq 1) { return @( " PackageVersion: $SqlClientPackageVersion" + ' FileVersion: 7.1.0.26238' ' PublishedVersion: 7.0.0' ) } return @( " PackageVersion: $SqlServerPackageVersion" + ' FileVersion: 1.1.0.26238' ' PublishedVersion: 1.0.0' ) }.GetNewClosure() @@ -62,108 +63,313 @@ BeforeAll { function Set-SuccessfulDotnetMock { Set-DotnetMock } -} -AfterAll { - Remove-Variable -Name 'computeVersionsDotnetCallCount' -Scope Global -ErrorAction SilentlyContinue -} + function Invoke-VersionTarget { + param( + [Parameter(Mandatory)] + [string]$Target, -Describe 'compute-versions.ps1 Effective Versions' { - BeforeEach { - Set-SuccessfulDotnetMock - } + [Parameter(Mandatory)] + [string]$NextVersionProperty, - It 'appends the build number after the prerelease suffix when package revisioning is disabled' { - $output = Invoke-ComputeVersions -AddRevision $false -BuildNumber $script:testBuildNumber + [Parameter(Mandatory)] + [string]$BaseVersion, - $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" - $output | Should -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0-preview1\.$script:testBuildNumberPattern" - $output | Should -Match "VersionRevision;isOutput=true]$script:testFileVersionBuildNumber" + [string]$BuildSuffix + ) + + $arguments = @( + 'build' + $buildProjectPath + "-t:$Target" + '-v:m' + '-nologo' + "-p:BuildNumber=$script:testBuildNumber" + "-p:$NextVersionProperty=$BaseVersion" + ) + if ($BuildSuffix) { + $arguments += "-p:BuildSuffix=$BuildSuffix" + } + + $output = & dotnet @arguments 2>&1 | Out-String + if ($LASTEXITCODE -ne 0) { + throw "$Target failed with exit code ${LASTEXITCODE}:`n$output" + } + + $output } - It 'inserts the revision before prerelease suffixes for built packages' { - $output = Invoke-ComputeVersions + # Drives PrepareForBuild rather than the validation target directly, because the hook point is + # itself the thing under test: a check wired after the compile would pass a direct invocation. + # An explicit target framework is required, as PrepareForBuild is not valid on the outer + # cross-targeting build. + function Invoke-VersionValidation { + param( + [Parameter(Mandatory)] + [string]$ProjectPath, - $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.42-preview3' - $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0\.42-preview1' + [Parameter(Mandatory)] + [string]$TargetFramework, + + [string[]]$Properties = @() + ) + + $arguments = @( + 'build' + $ProjectPath + '-f' + $TargetFramework + '-t:PrepareForBuild' + '-v:m' + '-nologo' + ) + $Properties + + $output = & dotnet @arguments 2>&1 | Out-String + [pscustomobject]@{ ExitCode = $LASTEXITCODE; Output = $output } } - It 'retains the published SqlServer package when SqlServer is not built' { - $output = Invoke-ComputeVersions -BuildSqlServer $false + function Invoke-BuildProjTarget { + param( + [Parameter(Mandatory)] + [string]$Target, - $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.42-preview3' - $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0' - $output | Should -Not -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0\.42' + [string[]]$Properties = @() + ) + + $arguments = @( + 'build' + $buildProjectPath + "-t:$Target" + '-v:m' + '-nologo' + ) + $Properties + + $output = & dotnet @arguments 2>&1 | Out-String + [pscustomobject]@{ ExitCode = $LASTEXITCODE; Output = $output } } +} - It 'wraps revisions above 65535 and logs the mapping as information' { - $output = Invoke-ComputeVersions -Revision 65536 +AfterAll { + Remove-Variable -Name 'computeVersionsDotnetCallCount' -Scope Global -ErrorAction SilentlyContinue +} - $output | Should -Match 'Revision 65536.*wrapped to 1' - $output | Should -Not -Match 'task\.logissue type=warning' - $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.1-preview3' - $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0\.1-preview1' - $output | Should -Match 'VersionRevision;isOutput=true]1' +Describe 'compute-versions.ps1 Effective Versions' { + BeforeEach { + Set-SuccessfulDotnetMock } - It 'emits the build number rather than the wrapped revision when package revisioning is disabled' { - $output = Invoke-ComputeVersions -Revision 65536 -AddRevision $false -BuildNumber $script:testBuildNumber + It 'forwards the stamped prerelease versions from Versions.props' { + $output = Invoke-ComputeVersions - $output | Should -Not -Match 'task\.logissue type=warning' - $output | Should -Not -Match 'wrapped to' $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" $output | Should -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0-preview1\.$script:testBuildNumberPattern" - $output | Should -Match "VersionRevision;isOutput=true]$script:testFileVersionBuildNumber" + $output | Should -Match 'SqlClientApiScanVersion;isOutput=true]7\.1' + $output | Should -Match 'SqlServerApiScanVersion;isOutput=true]1\.1' + $output | Should -Match 'APIScan registration versions:\s+SqlClient \(family\): 7\.1\s+SqlServer:\s+1\.1' + $output | Should -Match 'SqlClientFileVersion;isOutput=true]7\.1\.0\.26238' + $output | Should -Match 'SqlServerFileVersion;isOutput=true]1\.1\.0\.26238' } - It 'retains the published SqlServer package unstamped when SqlServer is not built' { - $output = Invoke-ComputeVersions -BuildSqlServer $false -AddRevision $false -BuildNumber $script:testBuildNumber + It 'retains the published SqlServer package when SqlServer is not built' { + $output = Invoke-ComputeVersions -BuildSqlServer $false $output | Should -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0-preview3\.$script:testBuildNumberPattern" $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.0\.0' - $output | Should -Not -Match "SqlServerPackageVersion;isOutput=true]1\.0\.0\.$script:testFileVersionBuildNumber" + $output | Should -Match 'SqlServerApiScanVersion;isOutput=true]1\.0' + $output | Should -Not -Match "SqlServerPackageVersion;isOutput=true]1\.0\.0[\.-]$script:testFileVersionBuildNumber" + + # An unbuilt SqlServer is never stamped, so it has no effective file version. + $output | Should -Match 'SqlServerFileVersion;isOutput=true](\r?\n|$)' } - It 'omits the build number from non-preview package versions when package revisioning is disabled' { + It 'forwards unstamped non-preview package versions' { Set-DotnetMock -SqlClientPackageVersion '7.1.0' -SqlServerPackageVersion '1.1.0' - $output = Invoke-ComputeVersions -AddRevision $false -BuildNumber $script:testBuildNumber + $output = Invoke-ComputeVersions $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0(\r?\n|$)' $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0(\r?\n|$)' $output | Should -Not -Match "SqlClientPackageVersion;isOutput=true]7\.1\.0[\.-]$script:testFileVersionBuildNumber" $output | Should -Not -Match "SqlServerPackageVersion;isOutput=true]1\.1\.0[\.-]$script:testFileVersionBuildNumber" - # The file version is still stamped so every build produces a distinct, date-encoded - # file version even for non-preview releases. - $output | Should -Match "VersionRevision;isOutput=true]$script:testFileVersionBuildNumber" + # The file version is still stamped so every build produces a date-encoded file version even + # for non-preview releases. + $output | Should -Match 'SqlClientFileVersion;isOutput=true]7\.1\.0\.26238' } +} - It 'revises non-preview package versions when package revisioning is enabled' { - Set-DotnetMock -SqlClientPackageVersion '7.1.0' -SqlServerPackageVersion '1.1.0' - - $output = Invoke-ComputeVersions - - $output | Should -Match 'SqlClientPackageVersion;isOutput=true]7\.1\.0\.42' - $output | Should -Match 'SqlServerPackageVersion;isOutput=true]1\.1\.0\.42' - $output | Should -Match 'VersionRevision;isOutput=true]42' +Describe 'GetVersions target package composition' { + It ' composes package and file versions' -ForEach @( + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0'; BuildSuffix = ''; ExpectedPackageVersion = '7.1.0' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a stable base without a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0'; BuildSuffix = 'ci'; ExpectedPackageVersion = '7.1.0-ci.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a stable base with a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0-preview3'; BuildSuffix = ''; ExpectedPackageVersion = '7.1.0-preview3.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a prerelease base without a suffix' + } + @{ + Target = 'GetVersionsSqlClient'; NextVersionProperty = 'SqlClientNextVersion' + BaseVersion = '7.1.0-preview3'; BuildSuffix = 'ci'; ExpectedPackageVersion = '7.1.0-preview3-ci.26238.3' + ExpectedFileVersion = '7.1.0.26238'; Case = 'a prerelease base with a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0'; BuildSuffix = ''; ExpectedPackageVersion = '1.1.0' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a stable base without a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0'; BuildSuffix = 'ci'; ExpectedPackageVersion = '1.1.0-ci.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a stable base with a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0-preview1'; BuildSuffix = ''; ExpectedPackageVersion = '1.1.0-preview1.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a prerelease base without a suffix' + } + @{ + Target = 'GetVersionsSqlServer'; NextVersionProperty = 'SqlServerNextVersion' + BaseVersion = '1.1.0-preview1'; BuildSuffix = 'ci'; ExpectedPackageVersion = '1.1.0-preview1-ci.26238.3' + ExpectedFileVersion = '1.1.0.26238'; Case = 'a prerelease base with a suffix' + } + ) { + $output = Invoke-VersionTarget ` + -Target $Target ` + -NextVersionProperty $NextVersionProperty ` + -BaseVersion $BaseVersion ` + -BuildSuffix $BuildSuffix + + $output | Should -Match "PackageVersion:\s+$([regex]::Escape($ExpectedPackageVersion))(\r?\n|$)" + $output | Should -Match "FileVersion:\s+$([regex]::Escape($ExpectedFileVersion))(\r?\n|$)" } } -Describe 'compute-versions.ps1 Error Handling' { - It 'rejects a non-positive revision' { - { Invoke-ComputeVersions -Revision 0 } | Should -Throw +Describe 'File version component validation' { + It 'rejects a four-part ' -ForEach @( + @{ + Product = 'SqlClient'; Property = 'SqlClientPackageVersion' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + Properties = @('-p:SqlClientPackageVersion=7.1.0.123') + ExpectedFileVersion = '7.1.0.123.0' + } + @{ + Product = 'SqlClient'; Property = 'SqlClientNextVersion' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + Properties = @('-p:SqlClientNextVersion=7.1.0.123', '-p:BuildNumber=1234') + ExpectedFileVersion = '7.1.0.123.1234' + } + @{ + Product = 'SqlServer'; Property = 'SqlServerPackageVersion' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + Properties = @('-p:SqlServerPackageVersion=1.1.0.123') + ExpectedFileVersion = '1.1.0.123.0' + } + @{ + Product = 'SqlServer'; Property = 'SqlServerNextVersion' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + Properties = @('-p:SqlServerNextVersion=1.1.0.123', '-p:BuildNumber=1234') + ExpectedFileVersion = '1.1.0.123.1234' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties $Properties + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '$ExpectedFileVersion' is not a four-part numeric version")) } - It 'requires a build number when package revisioning is disabled' { - Set-SuccessfulDotnetMock + It 'rejects an externally supplied file version for ' -ForEach @( + @{ + Product = 'SqlClient'; Description = 'short' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + FileVersion = '1.2' + } + @{ + Product = 'SqlClient'; Description = 'non-numeric' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + FileVersion = 'abc' + } + @{ + Product = 'SqlServer'; Description = 'short' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + FileVersion = '1.2' + } + @{ + Product = 'SqlServer'; Description = 'non-numeric' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + FileVersion = 'abc' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties @("-p:${Product}FileVersion=$FileVersion") + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '$FileVersion' is not a four-part numeric version")) + } + + It 'accepts the declared version' -ForEach @( + @{ + Product = 'SqlClient' + RelativeProject = 'src/Microsoft.Data.SqlClient/src/Microsoft.Data.SqlClient.csproj' + TargetFramework = 'net8.0' + } + @{ + Product = 'SqlServer' + RelativeProject = 'src/Microsoft.SqlServer.Server/Microsoft.SqlServer.Server.csproj' + TargetFramework = 'netstandard2.0' + } + ) { + $result = Invoke-VersionValidation ` + -ProjectPath (Join-Path $script:repoRoot $RelativeProject) ` + -TargetFramework $TargetFramework ` + -Properties @("-p:BuildNumber=$script:testBuildNumber") + + $result.ExitCode | Should -Be 0 + } +} - { Invoke-ComputeVersions -AddRevision $false } | - Should -Throw '*BuildNumber is required when AddRevision is false*' +Describe 'build.proj file version wrappers' { + # A malformed value is used so the leaf project reports it by name, which proves the wrapper + # forwarded it verbatim without paying for a full compile. + It 'forwards through ' -ForEach @( + @{ Product = 'SqlClient'; Wrapper = 'FileVersionSqlClient'; Target = 'BuildLogging' } + @{ Product = 'SqlServer'; Wrapper = 'FileVersionSqlServer'; Target = 'BuildSqlServer' } + ) { + $result = Invoke-BuildProjTarget -Target $Target -Properties @("-p:$Wrapper=1.2") + + $result.ExitCode | Should -Not -Be 0 + $result.Output | Should -Match ([regex]::Escape("${Product}FileVersion '1.2' is not a four-part numeric version")) } +} +Describe 'compute-versions.ps1 Error Handling' { It 'rejects a malformed build number' { - { Invoke-ComputeVersions -AddRevision $false -BuildNumber 'not-a-build-number' } | Should -Throw + { Invoke-ComputeVersions -BuildNumber 'not-a-build-number' } | Should -Throw + } + + It 'requires a build number' { + # Bound as empty rather than omitted; omitting a mandatory parameter prompts interactively. + { Invoke-ComputeVersions -BuildNumber '' } | Should -Throw } It 'throws when a GetVersions target fails' { @@ -184,18 +390,12 @@ Describe 'compute-versions.ps1 Error Handling' { { Invoke-ComputeVersions } | Should -Throw '*Failed to extract PackageVersion*' } - It 'throws when a revised package does not have a three-part numeric base' { - $global:computeVersionsDotnetCallCount = 0 + It 'throws when a FileVersion label is absent' { Mock -CommandName 'dotnet' -MockWith { $global:LASTEXITCODE = 0 - $global:computeVersionsDotnetCallCount++ - if ($global:computeVersionsDotnetCallCount -eq 1) { - return @('PackageVersion: 7.1-preview3') - } - - return @('PackageVersion: 1.1.0-preview1', 'PublishedVersion: 1.0.0') + return 'PackageVersion: 7.1.0-preview3.26238.3' } - { Invoke-ComputeVersions } | Should -Throw "*Expected a three-part numeric version base*" + { Invoke-ComputeVersions } | Should -Throw '*Failed to extract FileVersion*' } } diff --git a/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 new file mode 100644 index 0000000000..5f27b883d3 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/validate-localization.Tests.ps1 @@ -0,0 +1,190 @@ +<# +.SYNOPSIS + Pester tests for validate-localization.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'validate-localization.ps1' + + function Set-ResourceFile { + param( + [Parameter(Mandatory)][string]$Path, + [Parameter(Mandatory)][hashtable]$Strings + ) + + $document = [System.Xml.XmlDocument]::new() + $root = $document.CreateElement('root') + $null = $document.AppendChild($root) + foreach ($entry in $Strings.GetEnumerator()) { + $data = $document.CreateElement('data') + $data.SetAttribute('name', $entry.Key) + + $value = $document.CreateElement('value') + $value.InnerText = $entry.Value + $null = $data.AppendChild($value) + $null = $root.AppendChild($data) + } + + $document.Save($Path) + } + + function New-ResourcesDirectory { + $path = Join-Path $TestDrive ([guid]::NewGuid().ToString('n')) + New-Item -ItemType Directory -Path $path | Out-Null + return $path + } +} + +Describe 'validate-localization.ps1' { + It 'accepts complete localized files with translated values' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour'; Farewell = 'Au revoir' } + + { & $scriptPath -ResourcesDirectory $resources } | Should -Not -Throw + } + + It 'fails when a localized file is missing an English key' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.de.resx') @{ Greeting = 'Hallo' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when a localized value matches a non-empty English value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.ja.resx') @{ Greeting = 'Hello'; Unused = '' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when no localized resource files exist' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*No localized Strings.*.resx files were found*' + } + + It 'fails when a non-empty English string has an empty localized value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.es.resx') @{ Greeting = ' ' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'accepts empty localized values when the English value is also empty' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.ko.resx') @{ Unused = '' } + + { & $scriptPath -ResourcesDirectory $resources } | Should -Not -Throw + } + + It 'fails when a resource data element has no value' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + [xml]$localized = Get-Content -LiteralPath (Join-Path $resources 'Strings.fr.resx') + $valueNode = $localized.SelectSingleNode('/root/data/value') + $null = $valueNode.ParentNode.RemoveChild($valueNode) + $localized.Save((Join-Path $resources 'Strings.fr.resx')) + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*contains a element without a name or value*' + } + + It 'accepts an approved English-value match from the allowlist' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Not -Throw + } + + It 'does not allowlist a missing localized key' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello'; Farewell = 'Goodbye' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Farewell') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects allowlist for unknown resource keys' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Unknown') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'scopes approved English-value matches to one localized file' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.de.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Hello' } + @{ AllowedEnglishValueMatches = @{ 'Strings.de.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'fails when a localized file contains a key absent from Strings.resx' { + $resources = New-ResourcesDirectory + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour'; Obsolete = 'Ancien' } + + { & $scriptPath -ResourcesDirectory $resources } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects an allowlist entry after the localized value is translated' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Greeting = 'Hello' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Greeting = 'Bonjour' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Greeting') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*Localization validation failed with 1 error. Review the preceding errors.*' + } + + It 'rejects allowlist entries for empty English values' { + $resources = New-ResourcesDirectory + $allowlist = Join-Path $resources 'allowlist.json' + Set-ResourceFile (Join-Path $resources 'Strings.resx') @{ Unused = '' } + Set-ResourceFile (Join-Path $resources 'Strings.fr.resx') @{ Unused = '' } + @{ AllowedEnglishValueMatches = @{ 'Strings.fr.resx' = @('Unused') } } | + ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath $allowlist + + { & $scriptPath -ResourcesDirectory $resources -AllowlistPath $allowlist } | + Should -Throw '*does not have a non-empty English value*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 new file mode 100644 index 0000000000..626dd2dca9 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/validate-packages.Tests.ps1 @@ -0,0 +1,188 @@ +<# +.SYNOPSIS + Pester tests for validate-packages.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'validate-packages.ps1' + + # Stands in for the built PackageValidator.dll; the script only checks that it exists. + $script:validatorPath = Join-Path $TestDrive 'PackageValidator.dll' + Set-Content -LiteralPath $script:validatorPath -Value 'stub' + + $script:packagesPath = Join-Path $TestDrive 'packages' + New-Item -ItemType Directory -Force -Path $script:packagesPath | Out-Null + Set-Content -LiteralPath (Join-Path $script:packagesPath 'Microsoft.Data.SqlClient.7.1.0.nupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'Microsoft.SqlServer.Server.1.1.0.nupkg') -Value 'stub' + + $script:reportPath = Join-Path $TestDrive 'out' 'report.json' + + function Invoke-ValidatePackages { + param( + [string]$PackagesPath = $script:packagesPath, + [string]$ValidatorPath = $script:validatorPath, + [string]$SqlClientPackageVersion = '7.1.0-preview3.26238.3', + [string]$SqlClientFileVersion = '7.1.0.26238', + [string]$SqlServerPackageVersion = '', + [string]$SqlServerFileVersion = '', + [string[]]$FailOn = @('error') + ) + + & $scriptPath ` + -ValidatorPath $ValidatorPath ` + -PackagesPath $PackagesPath ` + -ReportPath $script:reportPath ` + -SqlClientPackageVersion $SqlClientPackageVersion ` + -SqlClientFileVersion $SqlClientFileVersion ` + -SqlServerPackageVersion $SqlServerPackageVersion ` + -SqlServerFileVersion $SqlServerFileVersion ` + -FailOn $FailOn ` + -DotnetPath 'dotnet' *>&1 | Out-String + } + + # Captures the arguments of each invocation so tests can assert on what the validator was + # asked to do, and controls the exit code of each run. + function Set-DotnetMock { + param( + [int]$GateExitCode = 0, + [int]$ReportExitCode = 0 + ) + + $global:validatePackagesInvocations = @() + Mock -CommandName 'dotnet' -MockWith { + $global:validatePackagesInvocations += , @($args) + # The first run carries --json and never gates; the second applies the gate. + if ($args -contains '--json') { + $global:LASTEXITCODE = $ReportExitCode + return '{ "packages": [], "summary": {} }' + } + + $global:LASTEXITCODE = $GateExitCode + return 'validator output' + }.GetNewClosure() + } +} + +AfterAll { + Remove-Variable -Name 'validatePackagesInvocations' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'validate-packages.ps1 Expectations' { + BeforeEach { + Set-DotnetMock + } + + It 'applies the SqlClient family versions as wildcard expectations' { + Invoke-ValidatePackages | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $gateArgs | Should -Contain '*=7.1.0-preview3.26238.3' + $gateArgs | Should -Contain '*=7.1.0.26238' + } + + It 'omits SqlServer expectations when its versions are not supplied' { + Invoke-ValidatePackages | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + ($gateArgs -join ' ') | Should -Not -Match 'Microsoft\.SqlServer\.Server=' + } + + It 'adds SqlServer expectations as a per-id override when supplied' { + Invoke-ValidatePackages -SqlServerPackageVersion '1.1.0-preview1.26238.3' -SqlServerFileVersion '1.1.0.26238' | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $gateArgs | Should -Contain 'Microsoft.SqlServer.Server=1.1.0-preview1.26238.3' + $gateArgs | Should -Contain 'Microsoft.SqlServer.Server=1.1.0.26238' + } + + It 'passes each gate token as its own --fail-on argument' { + Invoke-ValidatePackages -FailOn @('error', 'missing-symbols', 'package-unsigned') | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $joined = $gateArgs -join ' ' + $joined | Should -Match '--fail-on error' + $joined | Should -Match '--fail-on missing-symbols' + $joined | Should -Match '--fail-on package-unsigned' + } + + It 'splits a single comma-separated gate token, as an Azure Pipelines argument line supplies it' { + Invoke-ValidatePackages -FailOn 'error, missing-symbols' | Out-Null + + $gateArgs = $global:validatePackagesInvocations | Where-Object { $_ -notcontains '--json' } | Select-Object -First 1 + $joined = $gateArgs -join ' ' + $joined | Should -Match '--fail-on error' + $joined | Should -Match '--fail-on missing-symbols' + $joined | Should -Not -Match 'error,' + } + + It 'writes the JSON report before applying the gate' { + Invoke-ValidatePackages | Out-Null + + # The reporting run must come first so the report survives a failing gate. + $firstInvocation = $global:validatePackagesInvocations | Select-Object -First 1 + $firstInvocation | Should -Contain '--json' + Test-Path -LiteralPath $script:reportPath | Should -BeTrue + } + + It 'does not gate the reporting run' { + Invoke-ValidatePackages -FailOn @('error') | Out-Null + + $reportArgs = $global:validatePackagesInvocations | Where-Object { $_ -contains '--json' } | Select-Object -First 1 + ($reportArgs -join ' ') | Should -Not -Match '--fail-on' + } +} + +Describe 'validate-packages.ps1 Exit Codes' { + It 'succeeds when the validator reports no gating findings' { + Set-DotnetMock -GateExitCode 0 + + $output = Invoke-ValidatePackages + $output | Should -Match 'Package validation passed' + } + + It 'fails when a gate is tripped' { + Set-DotnetMock -GateExitCode 2 + + { Invoke-ValidatePackages -FailOn @('error', 'missing-symbols') } | + Should -Throw '*matched the gate (error, missing-symbols)*' + } + + It 'reports an unexpected validator failure distinctly from a tripped gate' { + Set-DotnetMock -GateExitCode 1 + + { Invoke-ValidatePackages } | Should -Throw '*exited unexpectedly with code 1*' + } + + It 'fails the reporting run before gating so the real cause is not obscured' { + Set-DotnetMock -ReportExitCode 1 + + { Invoke-ValidatePackages } | Should -Throw '*failed while writing the JSON report (exit code 1)*' + + # The gating run must not have been reached. + $global:validatePackagesInvocations.Count | Should -Be 1 + } +} + +Describe 'validate-packages.ps1 Error Handling' { + BeforeEach { + Set-DotnetMock + } + + It 'throws when the validator is missing' { + { Invoke-ValidatePackages -ValidatorPath (Join-Path $TestDrive 'absent.dll') } | + Should -Throw '*PackageValidator was not found*' + } + + It 'throws when no packages are found' { + $empty = Join-Path $TestDrive 'empty' + New-Item -ItemType Directory -Force -Path $empty | Out-Null + + { Invoke-ValidatePackages -PackagesPath $empty } | Should -Throw '*No .nupkg files were found*' + } + + It 'rejects a half-supplied SqlServer expectation' { + # Supplying only one would assert a package version without its file version. + { Invoke-ValidatePackages -SqlServerPackageVersion '1.1.0' } | + Should -Throw '*must be supplied together*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 new file mode 100644 index 0000000000..4d4946a5fb --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/verify-assembly-signatures.Tests.ps1 @@ -0,0 +1,149 @@ +<# +.SYNOPSIS + Pester tests for verify-assembly-signatures.ps1. + +.NOTES + Get-AuthenticodeSignature is a Windows-only cmdlet, so a stub is declared when it is absent. + This lets the tests run on any platform while still exercising the script's real expansion and + reporting logic; only the signature lookup itself is substituted. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'verify-assembly-signatures.ps1' + + if (-not (Get-Command 'Get-AuthenticodeSignature' -ErrorAction SilentlyContinue)) { + function Get-AuthenticodeSignature { + param([Parameter(Mandatory = $true)][string[]]$FilePath) + throw 'stub must be mocked' + } + } + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + # Builds a real .nupkg so the script's expansion path is genuinely exercised. + function New-TestPackage { + param( + [string]$Name, + [string[]]$AssemblyPaths = @('lib/net8.0/Test.dll'), + [string[]]$OtherPaths = @() + ) + + $staging = Join-Path $TestDrive "staging-$Name" + if (Test-Path -LiteralPath $staging) { Remove-Item -LiteralPath $staging -Recurse -Force } + New-Item -ItemType Directory -Force -Path $staging | Out-Null + + foreach ($relative in ($AssemblyPaths + $OtherPaths)) { + $full = Join-Path $staging $relative + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $full) | Out-Null + Set-Content -LiteralPath $full -Value 'stub' + } + + $packagePath = Join-Path $script:packagesPath "$Name.nupkg" + if (Test-Path -LiteralPath $packagePath) { Remove-Item -LiteralPath $packagePath -Force } + [System.IO.Compression.ZipFile]::CreateFromDirectory($staging, $packagePath) + return $packagePath + } + + function Invoke-VerifyAssemblySignatures { + param( + [string]$PackagesPath = $script:packagesPath, + [string]$ExtractPath = $script:extractPath + ) + + & $scriptPath -PackagesPath $PackagesPath -ExtractPath $ExtractPath *>&1 | Out-String + } +} + +AfterAll { + Remove-Variable -Name 'verifyAssemblySeen' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'verify-assembly-signatures.ps1' { + BeforeEach { + $script:packagesPath = Join-Path $TestDrive 'packages' + $script:extractPath = Join-Path $TestDrive 'extract' + foreach ($path in @($script:packagesPath, $script:extractPath)) { + if (Test-Path -LiteralPath $path) { Remove-Item -LiteralPath $path -Recurse -Force } + New-Item -ItemType Directory -Force -Path $path | Out-Null + } + } + + It 'expands packages and verifies every assembly they contain' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') | Out-Null + New-TestPackage -Name 'PackageTwo' -AssemblyPaths @('lib/net8.0/Two.dll', 'runtimes/win-x64/native/sni.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 3 assemblies are Authenticode signed' + } + + It 'finds native binaries under runtimes, not just managed assemblies' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('runtimes/win-arm64/native/sni.dll') | Out-Null + $global:verifyAssemblySeen = @() + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $global:verifyAssemblySeen = $FilePath + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + Invoke-VerifyAssemblySignatures | Out-Null + ($global:verifyAssemblySeen -join ';') | Should -Match 'sni\.dll' + } + + It 'ignores non-assembly content' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') -OtherPaths @('README.md', 'lib/net8.0/One.xml') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 1 assemblies are Authenticode signed' + } + + It 'fails when an assembly is not validly signed' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Two.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $index = 0 + $FilePath | ForEach-Object { + $status = if ($index -eq 0) { 'Valid' } else { 'NotSigned' } + $index++ + [pscustomobject]@{ Path = $_; Status = $status } + } + } + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*failed for 1 of 2 assemblies*' + } + + It 'reports every unsigned assembly rather than stopping at the first' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Two.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'NotSigned' } } + } + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*failed for 2 of 2 assemblies*' + } + + It 'replaces a previous expansion so stale content cannot be verified' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll', 'lib/net8.0/Stale.dll') | Out-Null + Mock -CommandName 'Get-AuthenticodeSignature' -MockWith { + $FilePath | ForEach-Object { [pscustomobject]@{ Path = $_; Status = 'Valid' } } + } + Invoke-VerifyAssemblySignatures | Out-Null + + # Repack the same package id with fewer assemblies; the stale one must not linger. + New-TestPackage -Name 'PackageOne' -AssemblyPaths @('lib/net8.0/One.dll') | Out-Null + $output = Invoke-VerifyAssemblySignatures + $output | Should -Match 'All 1 assemblies are Authenticode signed' + } + + It 'throws when no packages are found' { + { Invoke-VerifyAssemblySignatures } | Should -Throw '*No .nupkg files were found*' + } + + It 'throws when packages contain no assemblies' { + New-TestPackage -Name 'PackageOne' -AssemblyPaths @() -OtherPaths @('README.md') | Out-Null + + { Invoke-VerifyAssemblySignatures } | Should -Throw '*No assemblies were found*' + } +} diff --git a/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 b/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 new file mode 100644 index 0000000000..cadc1831a4 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/tests/verify-package-signatures.Tests.ps1 @@ -0,0 +1,87 @@ +<# +.SYNOPSIS + Pester tests for verify-package-signatures.ps1. +#> + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot '..' 'verify-package-signatures.ps1' + + $script:packagesPath = Join-Path $TestDrive 'packages' + New-Item -ItemType Directory -Force -Path (Join-Path $script:packagesPath 'SqlClient') | Out-Null + New-Item -ItemType Directory -Force -Path (Join-Path $script:packagesPath 'SqlServer') | Out-Null + + # Symbol packages are signed too, so both extensions must be picked up, and the nested layout + # mirrors how each artifact is downloaded into its own subdirectory. + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlClient' 'Microsoft.Data.SqlClient.7.1.0.nupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlClient' 'Microsoft.Data.SqlClient.7.1.0.snupkg') -Value 'stub' + Set-Content -LiteralPath (Join-Path $script:packagesPath 'SqlServer' 'Microsoft.SqlServer.Server.1.1.0.nupkg') -Value 'stub' + + function Invoke-VerifyPackageSignatures { + param([string]$PackagesPath = $script:packagesPath) + + & $scriptPath -PackagesPath $PackagesPath -DotnetPath 'dotnet' *>&1 | Out-String + } + + # Fails verification only for packages whose name matches, so tests can make a subset unsigned. + function Set-DotnetMock { + param([string]$FailPattern = '') + + $global:verifyPackageInvocations = @() + Mock -CommandName 'dotnet' -MockWith { + $global:verifyPackageInvocations += , @($args) + $target = $args[-1] + if ($FailPattern -and $target -match $FailPattern) { + $global:LASTEXITCODE = 1 + return "unsigned" + } + + $global:LASTEXITCODE = 0 + return "verified" + }.GetNewClosure() + } +} + +AfterAll { + Remove-Variable -Name 'verifyPackageInvocations' -Scope Global -ErrorAction SilentlyContinue +} + +Describe 'verify-package-signatures.ps1' { + It 'verifies every package and symbol package found' { + Set-DotnetMock + + $output = Invoke-VerifyPackageSignatures + $output | Should -Match 'All 3 package signature\(s\) verified' + $global:verifyPackageInvocations.Count | Should -Be 3 + } + + It 'invokes dotnet nuget verify with --all' { + Set-DotnetMock + + Invoke-VerifyPackageSignatures | Out-Null + + $first = $global:verifyPackageInvocations | Select-Object -First 1 + ($first -join ' ') | Should -Match 'nuget verify --all' + } + + It 'fails when a package signature does not verify' { + Set-DotnetMock -FailPattern 'SqlServer' + + { Invoke-VerifyPackageSignatures } | Should -Throw '*Microsoft.SqlServer.Server.1.1.0.nupkg*' + } + + It 'checks every package before failing so all failures are reported' { + Set-DotnetMock -FailPattern '\.nupkg$' + + # Two of the three files are .nupkg; both must appear rather than only the first. + { Invoke-VerifyPackageSignatures } | Should -Throw '*failed for 2 of 3 package(s)*' + $global:verifyPackageInvocations.Count | Should -Be 3 + } + + It 'throws when no packages are found' { + Set-DotnetMock + $empty = Join-Path $TestDrive 'empty' + New-Item -ItemType Directory -Force -Path $empty | Out-Null + + { Invoke-VerifyPackageSignatures -PackagesPath $empty } | Should -Throw '*No package files were found*' + } +} diff --git a/eng/pipelines/onebranch/scripts/validate-localization.ps1 b/eng/pipelines/onebranch/scripts/validate-localization.ps1 new file mode 100644 index 0000000000..1922c78c5b --- /dev/null +++ b/eng/pipelines/onebranch/scripts/validate-localization.ps1 @@ -0,0 +1,180 @@ +<# +.SYNOPSIS + Validates localized Strings.*.resx files against Strings.resx. + +.PARAMETER ResourcesDirectory + Directory containing the English and localized Strings.resx files. + +.PARAMETER AllowlistPath + Optional JSON file containing approved English-value matches grouped by localized filename. +#> + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# See the LICENSE file in the project root for more information. + +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string]$ResourcesDirectory, + + [string]$AllowlistPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Get-ResourceStrings { + param([Parameter(Mandatory)][string]$Path) + + $document = [System.Xml.Linq.XDocument]::Load($Path) + $strings = [System.Collections.Generic.Dictionary[string, string]]::new([System.StringComparer]::Ordinal) + foreach ($data in $document.Root.Elements('data')) { + $name = $data.Attribute('name') + $value = $data.Element('value') + if ($null -eq $name -or $null -eq $value) { + throw "Resource file '$Path' contains a element without a name or value." + } + + if ($strings.ContainsKey($name.Value)) { + throw "Resource file '$Path' contains duplicate key '$($name.Value)'." + } + + $strings.Add($name.Value, $value.Value) + } + + return $strings +} + +# Discover the neutral English resource and every culture-specific resource. The English file is +# the source of truth for the required key set and value-comparison checks below. +$resourcesPath = (Resolve-Path -LiteralPath $ResourcesDirectory).Path +$englishPath = Join-Path $resourcesPath 'Strings.resx' +if (-not (Test-Path -LiteralPath $englishPath -PathType Leaf)) { + throw "English resource file '$englishPath' was not found." +} + +$localizedFiles = @(Get-ChildItem -LiteralPath $resourcesPath -Filter 'Strings.*.resx' -File | Sort-Object Name) +if ($localizedFiles.Count -eq 0) { + throw "No localized Strings.*.resx files were found in '$resourcesPath'." +} + +$englishStrings = Get-ResourceStrings -Path $englishPath +$localizedFilesByName = [System.Collections.Generic.Dictionary[string, System.IO.FileInfo]]::new([System.StringComparer]::Ordinal) +foreach ($localizedFile in $localizedFiles) { + $localizedFilesByName.Add($localizedFile.Name, $localizedFile) +} + +# Load culture/key-specific exceptions for translations intentionally identical to English. The +# configuration is validated against the current resources so stale filenames and keys fail the +# build instead of silently weakening future validation. +$failures = [System.Collections.Generic.List[string]]::new() +$allowedEnglishMatches = [System.Collections.Generic.Dictionary[string, System.Collections.Generic.HashSet[string]]]::new([System.StringComparer]::Ordinal) +if (-not [string]::IsNullOrWhiteSpace($AllowlistPath)) { + if (-not (Test-Path -LiteralPath $AllowlistPath -PathType Leaf)) { + throw "Localization allowlist file '$AllowlistPath' was not found." + } + + $configuration = Get-Content -LiteralPath $AllowlistPath -Raw | ConvertFrom-Json + $englishValueMatchesProperty = $configuration.PSObject.Properties['AllowedEnglishValueMatches'] + if ($null -eq $englishValueMatchesProperty) { + throw "Localization allowlist file '$AllowlistPath' must define 'AllowedEnglishValueMatches'." + } + + foreach ($fileProperty in $englishValueMatchesProperty.Value.PSObject.Properties) { + if (-not $localizedFilesByName.ContainsKey($fileProperty.Name)) { + $failures.Add("Localization allowlist file references unknown resource file '$($fileProperty.Name)'.") + continue + } + + $keys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($key in @($fileProperty.Value)) { + if ([string]::IsNullOrWhiteSpace($key)) { + throw "Localization allowlist file contains an empty resource key for '$($fileProperty.Name)'." + } + if (-not $englishStrings.ContainsKey($key)) { + $failures.Add("Localization allowlist file references unknown resource key '$key' for '$($fileProperty.Name)'.") + continue + } + if ([string]::IsNullOrEmpty($englishStrings[$key])) { + throw "Localization allowlist key '$key' for '$($fileProperty.Name)' does not have a non-empty English value." + } + if (-not $keys.Add($key)) { + throw "Localization allowlist file contains duplicate key '$key' for '$($fileProperty.Name)'." + } + } + $allowedEnglishMatches.Add($fileProperty.Name, $keys) + } +} + +$allowedMatchCount = 0 +foreach ($localizedFile in $localizedFiles) { + $localizedStrings = Get-ResourceStrings -Path $localizedFile.FullName + + # Validate the key sets in both directions, then compare every non-empty English value with + # its localized value. Empty neutral values may remain empty because they contain no text to + # translate. + $missingOrEmptyKeys = [System.Collections.Generic.List[string]]::new() + $localizedOnlyKeys = [System.Collections.Generic.List[string]]::new() + $englishMatches = [System.Collections.Generic.List[string]]::new() + $staleAllowlistKeys = [System.Collections.Generic.List[string]]::new() + + foreach ($localizedKey in $localizedStrings.Keys) { + if (-not $englishStrings.ContainsKey($localizedKey)) { + $localizedOnlyKeys.Add($localizedKey) + } + } + + foreach ($entry in $englishStrings.GetEnumerator()) { + if (-not $localizedStrings.ContainsKey($entry.Key) -or + (-not [string]::IsNullOrEmpty($entry.Value) -and + [string]::IsNullOrWhiteSpace($localizedStrings[$entry.Key]))) { + $missingOrEmptyKeys.Add($entry.Key) + } + elseif (-not [string]::IsNullOrEmpty($entry.Value) -and + [System.StringComparer]::Ordinal.Equals($entry.Value, $localizedStrings[$entry.Key])) { + if ($allowedEnglishMatches.ContainsKey($localizedFile.Name) -and + $allowedEnglishMatches[$localizedFile.Name].Contains($entry.Key)) { + $allowedMatchCount++ + } + else { + $englishMatches.Add($entry.Key) + } + } + # An allowlist entry must be removed after its localized value changes; otherwise a future + # regression to the English value could be hidden by an obsolete exception. + elseif ($allowedEnglishMatches.ContainsKey($localizedFile.Name) -and + $allowedEnglishMatches[$localizedFile.Name].Contains($entry.Key)) { + $staleAllowlistKeys.Add($entry.Key) + } + } + + if ($missingOrEmptyKeys.Count -gt 0) { + $missingOrEmptyKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): missing keys or empty values: $($missingOrEmptyKeys -join ', ')") + } + if ($localizedOnlyKeys.Count -gt 0) { + $localizedOnlyKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): keys not found in Strings.resx: $($localizedOnlyKeys -join ', ')") + } + if ($englishMatches.Count -gt 0) { + $englishMatches.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): untranslated values match Strings.resx: $($englishMatches -join ', ')") + } + if ($staleAllowlistKeys.Count -gt 0) { + $staleAllowlistKeys.Sort([System.StringComparer]::Ordinal) + $failures.Add("$($localizedFile.Name): allowlist entries no longer match Strings.resx: $($staleAllowlistKeys -join ', ')") + } +} + +if ($failures.Count -gt 0) { + foreach ($failure in $failures) { + Write-Host "##vso[task.logissue type=error]$failure" + } + $errorNoun = if ($failures.Count -eq 1) { 'error' } else { 'errors' } + throw "Localization validation failed with $($failures.Count) $errorNoun. Review the preceding errors." +} + +$fileNoun = if ($localizedFiles.Count -eq 1) { 'file' } else { 'files' } +Write-Host "Localization validation passed for $($localizedFiles.Count) localized $fileNoun. Resource keys checked: $($englishStrings.Count); approved English-value matches allowlisted: $allowedMatchCount." diff --git a/eng/pipelines/onebranch/scripts/validate-packages.ps1 b/eng/pipelines/onebranch/scripts/validate-packages.ps1 new file mode 100644 index 0000000000..af5149c9f5 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/validate-packages.ps1 @@ -0,0 +1,213 @@ +<# +.SYNOPSIS + Runs the PackageValidator tool over the NuGet packages produced by a OneBranch build. + +.DESCRIPTION + Invokes tools/PackageValidator once for a whole directory of packages rather than once per + package, because its most valuable checks are cross-package: every package in the SqlClient + family must carry the same version, and their inter-package dependency ranges must agree. + Validating one package at a time would silently skip all of those findings. + + The validator runs twice over the same inputs. The first run writes a machine-readable report + and never gates, so the report exists even when validation fails. The second run renders the + human-readable report and applies the gate, so a failing build shows its findings in its own + log rather than only in an artifact. + + Expected versions are supplied by the caller rather than derived here. The compute-versions + stage already computes every version the build stamps, and re-deriving them would reintroduce + the drift this validation exists to catch. + + Microsoft.SqlServer.Server is versioned separately from the SqlClient family, so its expected + versions are applied as a per-id override of the family wildcard. When it is not built in a + run, its package is absent from the drop and its expectations must be omitted entirely: the + validator rejects an expectation whose value is empty. + +.PARAMETER ValidatorPath + Path to the built PackageValidator.dll. Invoked through the managed assembly rather than the + native apphost so the same command works regardless of agent OS. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg files. Sibling .snupkg files must sit beside their + .nupkg for symbol matching to resolve, which is how the build jobs publish them. + +.PARAMETER ReportPath + Path of the JSON report to write. Parent directories are created as needed. + +.PARAMETER SqlClientPackageVersion + Package version expected of every package in the SqlClient family, applied as a wildcard. + Pointing every package at one value is what proves they agree, and also catches the case where + all of them are consistently wrong. + +.PARAMETER SqlClientFileVersion + Assembly file version expected of every assembly in the SqlClient family. + +.PARAMETER SqlServerPackageVersion + Package version expected of Microsoft.SqlServer.Server. Omit when SqlServer is not built. + +.PARAMETER SqlServerFileVersion + Assembly file version expected of Microsoft.SqlServer.Server. Omit when SqlServer is not built. + +.PARAMETER FailOn + Finding severities and/or categories that fail the build. Run the validator with --help to see + the available categories. Note that missing-symbols is a warning and package-unsigned is info, + so neither is covered by the error severity and both must be named explicitly. + + Accepts either an array or a single comma-separated string, because an Azure Pipelines task + argument line collapses to one token and PowerShell's -File mode does not split it. + +.PARAMETER DotnetPath + dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter + primarily supports isolated testing. + +.EXAMPLE + ./validate-packages.ps1 ` + -ValidatorPath ./PackageValidator.dll ` + -PackagesPath ./packages ` + -ReportPath ./out/report.json ` + -SqlClientPackageVersion 7.1.0-preview3.26238.3 ` + -SqlClientFileVersion 7.1.0.26238 ` + -FailOn error,missing-symbols + + Validates a family-only drop, failing on any error and on missing symbols. + +.NOTES + File Name : validate-packages.ps1 + Requires : PowerShell 7+ and the repository-pinned .NET SDK. + Called by : validate-packages-step.yml + + PackageValidator exit codes: + 0 - No gating findings. + 1 - The validator itself failed. + 2 - A --fail-on gate was tripped. +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Path to the built PackageValidator.dll.")] + [ValidateNotNullOrEmpty()] + [string]$ValidatorPath, + + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for .nupkg files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(Mandatory = $true, HelpMessage = "Path of the JSON report to write.")] + [ValidateNotNullOrEmpty()] + [string]$ReportPath, + + [Parameter(Mandatory = $true, HelpMessage = "Package version expected of the SqlClient family.")] + [ValidateNotNullOrEmpty()] + [string]$SqlClientPackageVersion, + + [Parameter(Mandatory = $true, HelpMessage = "File version expected of the SqlClient family.")] + [ValidateNotNullOrEmpty()] + [string]$SqlClientFileVersion, + + [Parameter(HelpMessage = "Package version expected of Microsoft.SqlServer.Server, when built.")] + [string]$SqlServerPackageVersion = "", + + [Parameter(HelpMessage = "File version expected of Microsoft.SqlServer.Server, when built.")] + [string]$SqlServerFileVersion = "", + + [Parameter(Mandatory = $true, HelpMessage = "Severities and/or categories that fail the build.")] + [ValidateNotNullOrEmpty()] + [string[]]$FailOn, + + [Parameter(HelpMessage = "dotnet executable to invoke.")] + [ValidateNotNullOrEmpty()] + [string]$DotnetPath = "dotnet" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +# Split on commas so a single "error,missing-symbols" token behaves like a two-element array. +$failOnTokens = @($FailOn -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) +if ($failOnTokens.Count -eq 0) { + throw "FailOn must name at least one severity or category." +} + +Write-Host "=== Validate Packages Parameters ===" +Write-Host "ValidatorPath: ${ValidatorPath}" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "ReportPath: ${ReportPath}" +Write-Host "SqlClientPackageVersion: ${SqlClientPackageVersion}" +Write-Host "SqlClientFileVersion: ${SqlClientFileVersion}" +Write-Host "SqlServerPackageVersion: ${SqlServerPackageVersion}" +Write-Host "SqlServerFileVersion: ${SqlServerFileVersion}" +Write-Host "FailOn: $($failOnTokens -join ', ')" +Write-Host "====================================" + +if (-not (Test-Path -LiteralPath $ValidatorPath)) { + throw "PackageValidator was not found at '${ValidatorPath}'." +} + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Filter *.nupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No .nupkg files were found under '${PackagesPath}'." +} + +Write-Host "Validating $($packages.Count) package(s):" +$packages | ForEach-Object { Write-Host " $($_.Name)" } + +# A bare value applies to every package; an id=value pair overrides it for that package only. +$expectations = @( + "--expect-package-version", "*=${SqlClientPackageVersion}" + "--expect-file-version", "*=${SqlClientFileVersion}" +) + +# Both SqlServer versions travel together: supplying only one would assert half a package. +$hasSqlServerPackageVersion = -not [string]::IsNullOrWhiteSpace($SqlServerPackageVersion) +$hasSqlServerFileVersion = -not [string]::IsNullOrWhiteSpace($SqlServerFileVersion) +if ($hasSqlServerPackageVersion -ne $hasSqlServerFileVersion) { + throw "SqlServerPackageVersion and SqlServerFileVersion must be supplied together, or not at all." +} + +if ($hasSqlServerPackageVersion) { + $expectations += @( + "--expect-package-version", "Microsoft.SqlServer.Server=${SqlServerPackageVersion}" + "--expect-file-version", "Microsoft.SqlServer.Server=${SqlServerFileVersion}" + ) +} + +$gate = @() +foreach ($token in $failOnTokens) { + $gate += @("--fail-on", $token) +} + +Write-Host "Expectations: $($expectations -join ' ')" +Write-Host "Gate: $($gate -join ' ')" + +$reportDirectory = Split-Path -Parent $ReportPath +if ($reportDirectory) { + New-Item -ItemType Directory -Force -Path $reportDirectory | Out-Null +} + +# Reported before gating so the JSON exists even for a failing run. +& $DotnetPath $ValidatorPath $PackagesPath --json @expectations | + Set-Content -LiteralPath $ReportPath -Encoding utf8 +$reportExitCode = $LASTEXITCODE + +# This run is ungated, so any non-zero code means the validator itself failed and the report it +# produced cannot be trusted. Fail here rather than let the gated run obscure the real cause. +if ($reportExitCode -ne 0) { + throw "PackageValidator failed while writing the JSON report (exit code ${reportExitCode})." +} + +Write-Host "Wrote JSON report to ${ReportPath}" + +Write-Host "" +Write-Host "=== Package validation report ===" +& $DotnetPath $ValidatorPath $PackagesPath @expectations @gate +$exitCode = $LASTEXITCODE + +if ($exitCode -eq 0) { + Write-Host "" + Write-Host "Package validation passed." +} +elseif ($exitCode -eq 2) { + throw "Package validation failed: one or more findings matched the gate ($($failOnTokens -join ', '))." +} +else { + throw "PackageValidator exited unexpectedly with code ${exitCode}." +} diff --git a/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 b/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 new file mode 100644 index 0000000000..76c1495602 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/verify-assembly-signatures.ps1 @@ -0,0 +1,99 @@ +<# +.SYNOPSIS + Verifies that every assembly shipped inside an official build's NuGet packages is Authenticode + signed. + +.DESCRIPTION + Expands each .nupkg beneath a directory and checks the Authenticode signature of every assembly + it contains, including native binaries under runtimes/. + + Packages are expanded rather than installed through NuGet so that every produced package is + covered without resolving dependencies, and so that the check does not depend on any single + package id. + + This complements PackageValidator, which reports strong-name state from assembly metadata + cross-platform. Authenticode verification requires the Windows trust store, so this script runs + only on Windows agents and only for official builds; non-official builds deliberately produce + unsigned assemblies. + + Every assembly is checked before failing, so a single run reports all unsigned assemblies + rather than stopping at the first. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg files to expand. + +.PARAMETER ExtractPath + Directory the packages are expanded into. Each package is expanded into its own subdirectory so + that identically-named assemblies from different packages cannot collide. Existing content for + a package is replaced. + +.EXAMPLE + ./verify-assembly-signatures.ps1 -PackagesPath ./packages -ExtractPath ./extract + + Expands every package beneath ./packages and verifies the signature of each assembly. + +.NOTES + File Name : verify-assembly-signatures.ps1 + Requires : PowerShell 7+ on Windows (Get-AuthenticodeSignature). + Called by : validate-packages-job.yml +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for .nupkg files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(Mandatory = $true, HelpMessage = "Directory the packages are expanded into.")] + [ValidateNotNullOrEmpty()] + [string]$ExtractPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +Write-Host "=== Verify Assembly Signatures Parameters ===" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "ExtractPath: ${ExtractPath}" +Write-Host "=============================================" + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Filter *.nupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No .nupkg files were found under '${PackagesPath}'." +} + +New-Item -ItemType Directory -Force -Path $ExtractPath | Out-Null + +Add-Type -AssemblyName System.IO.Compression.FileSystem +foreach ($package in $packages) { + $destination = Join-Path $ExtractPath $package.BaseName + if (Test-Path -LiteralPath $destination) { + Remove-Item -LiteralPath $destination -Recurse -Force + } + + Write-Host "Expanding $($package.Name)" + [System.IO.Compression.ZipFile]::ExtractToDirectory($package.FullName, $destination) +} + +$assemblies = @(Get-ChildItem -Path $ExtractPath -Recurse -File -Filter *.dll) +if ($assemblies.Count -eq 0) { + throw "No assemblies were found under '${ExtractPath}'." +} + +# Every assembly is checked before throwing so one run reports all failures. +$unsigned = @() +foreach ($signature in @(Get-AuthenticodeSignature -FilePath $assemblies.FullName)) { + if ($signature.Status -eq "Valid") { + Write-Host " OK $($signature.Path)" + } + else { + Write-Host " FAIL $($signature.Path) - $($signature.Status)" + $unsigned += $signature.Path + } +} + +if ($unsigned.Count -gt 0) { + throw "Authenticode verification failed for $($unsigned.Count) of $($assemblies.Count) assemblies." +} + +Write-Host "All $($assemblies.Count) assemblies are Authenticode signed." diff --git a/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 b/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 new file mode 100644 index 0000000000..74fa864c07 --- /dev/null +++ b/eng/pipelines/onebranch/scripts/verify-package-signatures.ps1 @@ -0,0 +1,72 @@ +<# +.SYNOPSIS + Verifies the NuGet signatures of every package produced by an official OneBranch build. + +.DESCRIPTION + Runs `dotnet nuget verify --all` over every .nupkg and .snupkg found beneath a directory, + confirming that each carries a valid, trusted signature. + + This complements PackageValidator, which reports signature *presence* from package metadata + cross-platform. Establishing that a signature is trusted requires the platform trust store, + which is why this runs separately and only on official builds. Non-official builds deliberately + produce unsigned packages, so verifying them would always fail. + + Every package is verified before failing, so a single run reports all unsigned packages rather + than stopping at the first. + +.PARAMETER PackagesPath + Directory scanned recursively for .nupkg and .snupkg files. + +.PARAMETER DotnetPath + dotnet executable to invoke. Defaults to the dotnet command resolved from PATH. This parameter + primarily supports isolated testing. + +.EXAMPLE + ./verify-package-signatures.ps1 -PackagesPath ./packages + + Verifies every package and symbol package beneath ./packages. + +.NOTES + File Name : verify-package-signatures.ps1 + Requires : PowerShell 7+ and the repository-pinned .NET SDK. + Called by : validate-packages-job.yml +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true, HelpMessage = "Directory scanned recursively for package files.")] + [ValidateNotNullOrEmpty()] + [string]$PackagesPath, + + [Parameter(HelpMessage = "dotnet executable to invoke.")] + [ValidateNotNullOrEmpty()] + [string]$DotnetPath = "dotnet" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +Write-Host "=== Verify Package Signatures Parameters ===" +Write-Host "PackagesPath: ${PackagesPath}" +Write-Host "============================================" + +$packages = @(Get-ChildItem -Path $PackagesPath -Recurse -File -Include *.nupkg, *.snupkg -ErrorAction SilentlyContinue) +if ($packages.Count -eq 0) { + throw "No package files were found under '${PackagesPath}'." +} + +# Every package is checked before throwing so one run reports all failures. +$failed = @() +foreach ($package in $packages) { + Write-Host "Verifying $($package.Name)" + & $DotnetPath nuget verify --all $package.FullName + if ($LASTEXITCODE -ne 0) { + $failed += $package.Name + } +} + +if ($failed.Count -gt 0) { + throw "NuGet signature verification failed for $($failed.Count) of $($packages.Count) package(s): $($failed -join ', ')" +} + +Write-Host "All $($packages.Count) package signature(s) verified." diff --git a/eng/pipelines/onebranch/sqlclient-non-official.yml b/eng/pipelines/onebranch/sqlclient-non-official.yml index 7eb7bfc6cc..88231ed15b 100644 --- a/eng/pipelines/onebranch/sqlclient-non-official.yml +++ b/eng/pipelines/onebranch/sqlclient-non-official.yml @@ -28,13 +28,6 @@ parameters: type: boolean default: false - # Append the Build.BuildId revision to package and file versions to reduce collisions when - # repeatedly publishing tests to symbol servers or the NuGet test feed. - - name: addRevision - displayName: Use the build ID as the revision version - type: boolean - default: false - # True to publish symbols to private and public servers. - name: publishSymbols displayName: Publish symbols @@ -94,8 +87,29 @@ extends: parameters: featureFlags: + # WindowsHostVersion selects the Windows *host VM* that our Windows build container runs on. + # This is a separate layer from the container image itself (WindowsContainerImage in + # onebranch-variables.yml): the host is the outer machine running the Docker engine, and the + # container is where our build steps actually execute. + # + # These two must be kept compatible. Windows containers can only run on a host whose OS + # version is compatible with the container's base image. A mismatch (e.g. a 2025 container on + # a 2022 host) fails to start the container unless Hyper-V isolation is forced. + WindowsHostVersion: + Version: 2025 + + # We do NOT set a LinuxHostVersion. Unlike Windows, Linux containers share the host kernel, + # so there is no host/container OS-version compatibility requirement to satisfy -- our Linux + # build image (LinuxContainerImage in onebranch-variables.yml) runs on the default OneBranch + # Linux host regardless of its distribution. + + # CDPx is OneBranch's predecessor build system. When EnableCDPxPAT is true (the OneBranch + # default), the governed templates inject a legacy CDPx Personal Access Token and its + # associated NuGet / Azure Artifacts authentication variables (CDP_DEFAULT_CLIENT_PAT, + # VSS_NUGET_ACCESSTOKEN, VSS_NUGET_URI_PREFIXES, etc.) into the build and Docker jobs so + # package restore against Azure DevOps feeds works without explicit auth. We don't rely on + # that legacy CDPx package-authentication path, so we disable it. EnableCDPxPAT: false - WindowsHostVersion: 1ESWindows2022 release: # This indicates the pipeline category to deploy Box products. See: @@ -119,6 +133,34 @@ extends: # globalSdl: + # BREAK SEVERITY + # + # The SDL analyzer tasks never fail on findings; they only fail if the tool itself crashes or + # is misconfigured. The build break comes from the Post Analysis (Guardian Break) task, which + # reads the tool logs and fails when a finding meets or exceeds a minimum severity threshold. + # See https://aka.ms/gdn-azdo-break. + # + # Guardian normalises every finding to Error, Warning or Note. The threshold is cumulative, + # so a lower name is STRICTER, not looser: + # + # Error break on Error <-- Guardian's default + # Warning break on Error + Warning + # Note break on Error + Warning + Note + # + # Two knobs control it, in increasing order of precedence: + # + # globalSdl.severity threshold for every tool (maps to GdnBreakPolicyMinSev) + # globalSdl..severity per-tool override of the global threshold + # ob_sdl__severity per-job variable; overrides both of the above + # + # We omit `severity` everywhere and keep the Error-only default. OneBranch accepts ONLY + # Error, Warning or Note at this layer -- there is no explicit "Default" value to write, so + # inheriting the default requires omitting the key. Consequently, spelling out + # `severity: Error` on a tool is NOT equivalent to omitting it: it pins that tool to Error + # even if globalSdl.severity is later tightened. + # + # https://eng.ms/docs/products/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows/customizesdlforcontainerbuilds + # Snapshot of the SDL analyzer findings that pre-existed the breakOnSdlError rollout, so # builds only break on NEW findings. Generated from the SDL analysis artifacts of a full # non-official run and kept under .config/ alongside the other SDL tool configs @@ -144,16 +186,10 @@ extends: # Use pre-release mode for non-official pipelines. modeType: prerelease - # We have a single "name" registered with APIScan for all of our packages. - # - # https://eng.ms/docs/products/apiscan/onboard/requirements/registersoftware - # - softwareName: Microsoft.Data.SqlClient - - # Similar to the software name, we have a single version registered as well. This has - # nothing to do with the NuGet package version. It is purely an APIScan registration - # value that points to our backend configuration. - versionNumber: $(ApiScanSoftwareVersion) + # The APIScan software name and version are NOT set here. Each package is registered with + # APIScan under its own name/version pair, so every build job sets ob_sdl_apiscan_softwareName + # and ob_sdl_apiscan_versionNumber for the package it builds (see build-buildproj-job.yml). + # Jobs that produce no assemblies disable APIScan instead, via ob_sdl_apiscan_enabled. # We want the standard level of logging. verbosityLevel: standard @@ -198,7 +234,13 @@ extends: break: ${{ parameters.breakOnSdlError }} roslyn: - # Note, requires RoslynAnalyzers task to be added as a separate step + # Enabling Roslyn SDL analysis here requires that our .NET builds _produce_ Roslyn findings. + # You will see this in the separate Roslyn build task. + # + # Note that the Roslyn-specific Guardian collector/sanitizer requires SARIF v1, so our + # analysis build deliberately emits v1. Other, generic Guardian tooling expects SARIF v2 and + # may log processing errors (for example, Post Analysis's SDL artifact report) even though + # Roslyn collection and Guardian policy ingestion succeed. enabled: true break: ${{ parameters.breakOnSdlError }} @@ -207,8 +249,12 @@ extends: sbom: enabled: true - packageName: 'Microsoft.Data.SqlClient' - packageVersion: '$(Build.BuildNumber)' + # OneBranch resolves these from globalSdl only -- there is no per-job form -- so they + # indirect through variables that each build job sets to the package it produces. Jobs + # that publish no packages disable SBOM via ob_sdl_sbom_enabled rather than defaulting + # these. See build-buildproj-job.yml. + packageName: '$(sbomPackageName)' + packageVersion: '$(sbomPackageVersion)' tsa: # TSA (Trust Services Automation) files SDL analysis findings as Azure DevOps bug work @@ -235,6 +281,7 @@ extends: # TSA here does not by itself force breaking -- breakOnSdlError is what controls whether # findings fail the build. enabled: false + # Keep this in sync with Official even though TSA is disabled here. configFile: '$(REPO_ROOT)/.config/tsaoptions.json' stages: @@ -247,7 +294,6 @@ extends: - template: /eng/pipelines/onebranch/stages/compute-versions-stage.yml@self parameters: - addRevision: ${{ parameters.addRevision }} buildSqlServer: ${{ parameters.buildSqlServer }} - template: /eng/pipelines/onebranch/stages/build-stages.yml@self diff --git a/eng/pipelines/onebranch/sqlclient-official.yml b/eng/pipelines/onebranch/sqlclient-official.yml index 38a8a75496..40b08c76c0 100644 --- a/eng/pipelines/onebranch/sqlclient-official.yml +++ b/eng/pipelines/onebranch/sqlclient-official.yml @@ -37,12 +37,6 @@ parameters: type: boolean default: false - # Append the Build.BuildId revision to package and file versions. - - name: addRevision - displayName: Use the build ID as the revision version - type: boolean - default: false - # True to publish symbols to private and public servers. - name: publishSymbols displayName: Publish symbols @@ -108,8 +102,29 @@ extends: parameters: featureFlags: + # WindowsHostVersion selects the Windows *host VM* that our Windows build container runs on. + # This is a separate layer from the container image itself (WindowsContainerImage in + # onebranch-variables.yml): the host is the outer machine running the Docker engine, and the + # container is where our build steps actually execute. + # + # These two must be kept compatible. Windows containers can only run on a host whose OS + # version is compatible with the container's base image. A mismatch (e.g. a 2025 container on + # a 2022 host) fails to start the container unless Hyper-V isolation is forced. + WindowsHostVersion: + Version: 2025 + + # We do NOT set a LinuxHostVersion. Unlike Windows, Linux containers share the host kernel, + # so there is no host/container OS-version compatibility requirement to satisfy -- our Linux + # build image (LinuxContainerImage in onebranch-variables.yml) runs on the default OneBranch + # Linux host regardless of its distribution. + + # CDPx is OneBranch's predecessor build system. When EnableCDPxPAT is true (the OneBranch + # default), the governed templates inject a legacy CDPx Personal Access Token and its + # associated NuGet / Azure Artifacts authentication variables (CDP_DEFAULT_CLIENT_PAT, + # VSS_NUGET_ACCESSTOKEN, VSS_NUGET_URI_PREFIXES, etc.) into the build and Docker jobs so + # package restore against Azure DevOps feeds works without explicit auth. We don't rely on + # that legacy CDPx package-authentication path, so we disable it. EnableCDPxPAT: false - WindowsHostVersion: 1ESWindows2022 release: # This indicates the pipeline category to deploy Box products. See: @@ -133,6 +148,34 @@ extends: # globalSdl: + # BREAK SEVERITY + # + # The SDL analyzer tasks never fail on findings; they only fail if the tool itself crashes or + # is misconfigured. The build break comes from the Post Analysis (Guardian Break) task, which + # reads the tool logs and fails when a finding meets or exceeds a minimum severity threshold. + # See https://aka.ms/gdn-azdo-break. + # + # Guardian normalises every finding to Error, Warning or Note. The threshold is cumulative, + # so a lower name is STRICTER, not looser: + # + # Error break on Error <-- Guardian's default + # Warning break on Error + Warning + # Note break on Error + Warning + Note + # + # Two knobs control it, in increasing order of precedence: + # + # globalSdl.severity threshold for every tool (maps to GdnBreakPolicyMinSev) + # globalSdl..severity per-tool override of the global threshold + # ob_sdl__severity per-job variable; overrides both of the above + # + # We omit `severity` everywhere and keep the Error-only default. OneBranch accepts ONLY + # Error, Warning or Note at this layer -- there is no explicit "Default" value to write, so + # inheriting the default requires omitting the key. Consequently, spelling out + # `severity: Error` on a tool is NOT equivalent to omitting it: it pins that tool to Error + # even if globalSdl.severity is later tightened. + # + # https://eng.ms/docs/products/onebranch/securitycompliancegovernanceandpolicies/sdlforcontainerizedworkflows/customizesdlforcontainerbuilds + # Snapshot of the SDL analyzer findings that pre-existed the breakOnSdlError rollout, so # builds only break on NEW findings. Generated from the SDL analysis artifacts of a full # non-official run and kept under .config/ alongside the other SDL tool configs @@ -158,16 +201,10 @@ extends: # Use release mode for official pipelines. modeType: release - # We have a single "name" registered with APIScan for all of our packages. - # - # https://eng.ms/docs/products/apiscan/onboard/requirements/registersoftware - # - softwareName: Microsoft.Data.SqlClient - - # Similar to the software name, we have a single version registered as well. This has - # nothing to do with the NuGet package version. It is purely an APIScan registration - # value that points to our backend configuration. - versionNumber: $(ApiScanSoftwareVersion) + # The APIScan software name and version are NOT set here. Each package is registered with + # APIScan under its own name/version pair, so every build job sets ob_sdl_apiscan_softwareName + # and ob_sdl_apiscan_versionNumber for the package it builds (see build-buildproj-job.yml). + # Jobs that produce no assemblies disable APIScan instead, via ob_sdl_apiscan_enabled. # We want the standard level of logging. verbosityLevel: standard @@ -212,7 +249,13 @@ extends: break: ${{ parameters.breakOnSdlError }} roslyn: - # Note, requires RoslynAnalyzers task to be added as a separate step + # Enabling Roslyn SDL analysis here requires that our .NET builds _produce_ Roslyn findings. + # You will see this in the separate Roslyn build task. + # + # Note that the Roslyn-specific Guardian collector/sanitizer requires SARIF v1, so our + # analysis build deliberately emits v1. Other, generic Guardian tooling expects SARIF v2 and + # may log processing errors (for example, Post Analysis's SDL artifact report) even though + # Roslyn collection and Guardian policy ingestion succeed. enabled: true break: ${{ parameters.breakOnSdlError }} @@ -221,8 +264,12 @@ extends: sbom: enabled: true - packageName: 'Microsoft.Data.SqlClient' - packageVersion: '$(Build.BuildNumber)' + # OneBranch resolves these from globalSdl only -- there is no per-job form -- so they + # indirect through variables that each build job sets to the package it produces. Jobs + # that publish no packages disable SBOM via ob_sdl_sbom_enabled rather than defaulting + # these. See build-buildproj-job.yml. + packageName: '$(sbomPackageName)' + packageVersion: '$(sbomPackageVersion)' tsa: # TSA (Trust Services Automation) files SDL analysis findings as Azure DevOps bug work @@ -257,7 +304,6 @@ extends: - template: /eng/pipelines/onebranch/stages/compute-versions-stage.yml@self parameters: - addRevision: ${{ parameters.addRevision }} buildSqlServer: ${{ parameters.buildSqlServer }} - template: /eng/pipelines/onebranch/stages/build-stages.yml@self diff --git a/eng/pipelines/onebranch/stages/build-stages.yml b/eng/pipelines/onebranch/stages/build-stages.yml index 49c6616b85..34f88bd9b3 100644 --- a/eng/pipelines/onebranch/stages/build-stages.yml +++ b/eng/pipelines/onebranch/stages/build-stages.yml @@ -80,12 +80,18 @@ stages: dependsOn: compute_versions variables: - - name: versionRevision - value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlServerFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerFileVersion'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] + - name: sqlServerApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerApiScanVersion'] ] jobs: # Build Microsoft.Data.SqlClient.Internal.Logging @@ -93,6 +99,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Internal.Logging/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Internal.Logging/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -102,7 +109,7 @@ stages: signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' dependencies: [] - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient.Internal.Logging' packageShortName: 'Logging' versionPropertySuffix: 'SqlClient' @@ -115,6 +122,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.SqlServer.Server/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.SqlServer.Server/pdbs' + apiScanSoftwareVersion: '$(sqlServerApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -124,7 +132,7 @@ stages: signingEsrpConnectedServiceName: '${{ parameters.signingEsrpConnectedServiceName }}' dependencies: [] - revision: '$(versionRevision)' + fileVersion: '$(sqlServerFileVersion)' packageFullName: 'Microsoft.SqlServer.Server' packageShortName: 'SqlServer' versionPropertySuffix: 'SqlServer' @@ -142,10 +150,12 @@ stages: - build_independent variables: - - name: versionRevision - value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] jobs: # Build Microsoft.Data.SqlClient.Extensions.Abstractions @@ -153,6 +163,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Abstractions/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Abstractions/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -165,7 +176,7 @@ stages: - artifactName: '${{ parameters.loggingArtifactsName }}' shortName: 'Logging' version: '$(sqlClientPackageVersion)' - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient.Extensions.Abstractions' packageShortName: 'Abstractions' versionPropertySuffix: 'SqlClient' @@ -183,12 +194,14 @@ stages: - build_abstractions variables: - - name: versionRevision - value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] jobs: # Build Microsoft.Data.SqlClient @@ -196,6 +209,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -222,7 +236,7 @@ stages: - artifactName: '' shortName: 'SqlServer' version: '$(sqlServerPackageVersion)' - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient' packageShortName: 'SqlClient' versionPropertySuffix: 'SqlClient' @@ -233,6 +247,7 @@ stages: parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Azure/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.Extensions.Azure/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -248,7 +263,7 @@ stages: - artifactName: '${{ parameters.loggingArtifactsName }}' shortName: 'Logging' version: '$(sqlClientPackageVersion)' - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient.Extensions.Azure' packageShortName: 'Azure' versionPropertySuffix: 'SqlClient' @@ -265,18 +280,21 @@ stages: - build_dependent variables: - - name: versionRevision - value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.VersionRevision'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] - name: sqlClientPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] - name: sqlServerPackageVersion value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlClientApiScanVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientApiScanVersion'] ] jobs: - template: /eng/pipelines/onebranch/jobs/build-buildproj-job.yml@self parameters: apiScanDllPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/dlls' apiScanPdbPath: '$(REPO_ROOT)/apiScan/Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider/pdbs' + apiScanSoftwareVersion: '$(sqlClientApiScanVersion)' shouldSignPackage: ${{ parameters.isOfficial }} signingAppRegistrationClientId: '${{ parameters.signingAppRegistrationClientId }}' signingAppRegistrationTenantId: '${{ parameters.signingAppRegistrationTenantId }}' @@ -306,24 +324,51 @@ stages: - artifactName: '' shortName: 'SqlServer' version: '$(sqlServerPackageVersion)' - revision: '$(versionRevision)' + fileVersion: '$(sqlClientFileVersion)' packageFullName: 'Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider' packageShortName: 'AkvProvider' versionPropertySuffix: 'SqlClient' packageVersion: '$(sqlClientPackageVersion)' # ==================================================================== - # Validation - # @TODO: Update validate-signed-package-job to compute expected versions from - # Versions.props (same as build jobs) instead of receiving them as parameters. + # Stage 5: Validation + # Validates every package produced by this run, together, so that + # cross-package rules (shared family version, dependency agreement) + # are actually exercised. Depends on all build stages. # ==================================================================== - # - stage: sqlclient_package_validation - # displayName: "SqlClient Package Validation" - # dependsOn: build_dependent - # jobs: - # - template: /eng/pipelines/onebranch/jobs/validate-signed-package-job.yml@self - # parameters: - # artifactName: '${{ parameters.sqlClientArtifactsName }}' - # expectedFileVersion: - # expectedPackageVersion: - # isOfficial: ${{ parameters.isOfficial }} + - stage: package_validation + displayName: "Validate Packages" + dependsOn: + - compute_versions + - build_independent + - build_abstractions + - build_dependent + - build_addons + + variables: + - name: sqlClientPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientPackageVersion'] ] + - name: sqlClientFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlClientFileVersion'] ] + - name: sqlServerPackageVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerPackageVersion'] ] + - name: sqlServerFileVersion + value: $[ stageDependencies.compute_versions.compute_versions_job.outputs['versions.SqlServerFileVersion'] ] + + jobs: + - template: /eng/pipelines/onebranch/jobs/validate-packages-job.yml@self + parameters: + abstractionsArtifactsName: '${{ parameters.abstractionsArtifactsName }}' + akvProviderArtifactsName: '${{ parameters.akvProviderArtifactsName }}' + azureArtifactsName: '${{ parameters.azureArtifactsName }}' + loggingArtifactsName: '${{ parameters.loggingArtifactsName }}' + sqlClientArtifactsName: '${{ parameters.sqlClientArtifactsName }}' + sqlServerArtifactsName: '${{ parameters.sqlServerArtifactsName }}' + + sqlClientPackageVersion: '$(sqlClientPackageVersion)' + sqlClientFileVersion: '$(sqlClientFileVersion)' + sqlServerPackageVersion: '$(sqlServerPackageVersion)' + sqlServerFileVersion: '$(sqlServerFileVersion)' + + buildSqlServer: ${{ parameters.buildSqlServer }} + isOfficial: ${{ parameters.isOfficial }} diff --git a/eng/pipelines/onebranch/stages/compute-versions-stage.yml b/eng/pipelines/onebranch/stages/compute-versions-stage.yml index 11e2187e57..9514c0d56e 100644 --- a/eng/pipelines/onebranch/stages/compute-versions-stage.yml +++ b/eng/pipelines/onebranch/stages/compute-versions-stage.yml @@ -17,20 +17,14 @@ # - Microsoft.SqlServer.Server is versioned separately: it uses its "next" version when it is # being built this run, or its "published" (last shipped to NuGet) version when it is not built # (so the SqlClient family depends on the most recently published SqlServer package). -# - Runs use their "next" version from Versions.props. By default the pipeline build number is -# appended to prerelease package versions (7.1.0-preview3.26238.3) and its date segment becomes -# the file-version build number (7.1.0.26238), matching the shape shipped by earlier previews. -# Runs can instead append the Build.BuildId revision, reducing collisions when repeatedly -# publishing the same base version for testing. +# - Runs use their "next" version from Versions.props. The pipeline build number is appended to +# prerelease package versions (7.1.0-preview3.26238.3) and its date segment becomes the +# file-version build number (7.1.0.26238), matching the shape shipped by earlier previews. # - Versions are extracted via build.proj GetVersions* targets by parsing stdout. # # This stage MUST run before all build stages so they can consume computed versions. parameters: - # Whether to append the Build.BuildId revision to package and file versions. - - name: addRevision - type: boolean - # Whether Microsoft.SqlServer.Server is being built this run. This drives the SqlServer version # selection (next when built, published when not); the SqlClient family always uses its next # version. When SqlServer is not built, the SqlClient family depends on the published SqlServer @@ -54,6 +48,8 @@ stages: ob_sdl_apiscan_enabled: false ob_sdl_binskim_break: false ob_sdl_binskim_enabled: false + # No packages are published here, so there is nothing to describe in an SBOM. + ob_sdl_sbom_enabled: false steps: - pwsh: | @@ -69,8 +65,8 @@ stages: # may run during the GetVersions* evaluation. - template: /eng/pipelines/common/steps/restore-dotnet-tools.yml@self - # Extract canonical versions, resolve the effective built/published values, optionally - # add the run revision, and publish the downstream stage output variables. + # Extract canonical versions, resolve the effective built/published values, and publish + # the downstream stage output variables. - task: PowerShell@2 displayName: "Compute Effective Versions" name: "versions" @@ -80,7 +76,5 @@ stages: filePath: $(Build.SourcesDirectory)/eng/pipelines/onebranch/scripts/compute-versions.ps1 arguments: >- -ProjectPath "$(Build.SourcesDirectory)/build.proj" - -Revision "$(Build.BuildId)" -BuildNumber "$(Build.BuildNumber)" -BuildSqlServer $${{ parameters.buildSqlServer }} - -AddRevision $${{ parameters.addRevision }} diff --git a/eng/pipelines/onebranch/stages/release-stages.yml b/eng/pipelines/onebranch/stages/release-stages.yml index d160c8906f..7ee8023cd9 100644 --- a/eng/pipelines/onebranch/stages/release-stages.yml +++ b/eng/pipelines/onebranch/stages/release-stages.yml @@ -112,6 +112,10 @@ stages: ${{ else }}: displayName: Release to NuGet Test dependsOn: + # Nothing is published unless every produced package passed validation. This stage also + # depends on all four build stages, but we keep the complete list here anyway to prevent + # regressions if package validation changes its dependencies. + - package_validation - ${{ if or(parameters.releaseSqlServer, parameters.releaseSqlClient) }}: - build_independent - ${{ if parameters.releaseSqlClient }}: diff --git a/eng/pipelines/onebranch/steps/build-buildproj-step.yml b/eng/pipelines/onebranch/steps/build-buildproj-step.yml index 5cfc809772..961b972f8c 100644 --- a/eng/pipelines/onebranch/steps/build-buildproj-step.yml +++ b/eng/pipelines/onebranch/steps/build-buildproj-step.yml @@ -37,8 +37,8 @@ parameters: - SqlClient - SqlServer - # Version revision translated to build.proj's BuildNumber property at this boundary. - - name: revision + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion type: string # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this @@ -75,7 +75,7 @@ steps: -p:ReferenceType=Package -p:SkipDependencyPack=true -p:SigningKeyPath="$(keyFile.secureFilePath)" - -p:BuildNumber="${{ parameters.revision }}" + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" ${{ parameters.dependencyArguments }} diff --git a/eng/pipelines/onebranch/steps/pack-buildproj-step.yml b/eng/pipelines/onebranch/steps/pack-buildproj-step.yml index 4960872d7b..b8ee6e72dd 100644 --- a/eng/pipelines/onebranch/steps/pack-buildproj-step.yml +++ b/eng/pipelines/onebranch/steps/pack-buildproj-step.yml @@ -49,8 +49,8 @@ parameters: - SqlClient - SqlServer - # Version revision translated to build.proj's BuildNumber property at this boundary. - - name: revision + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion type: string # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this @@ -79,7 +79,7 @@ steps: -p:Configuration=${{ parameters.buildConfiguration }} -p:PackBuild=false -p:ReferenceType=Package - -p:BuildNumber="${{ parameters.revision }}" + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" ${{ parameters.dependencyArguments }} diff --git a/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml b/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml index 9e640b00df..6e9b0e8106 100644 --- a/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml +++ b/eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml @@ -4,13 +4,111 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This template defines a step to run Roslyn Analyzers on the SqlClient build. It uses the -# RoslynAnalyzers@3 task from the Secure Development Team's SDL extension: +# This template runs Roslyn Analyzers (SDL) against a build.proj target using the RoslynAnalyzers@3 +# task from the Secure Development Team's SDL extension, in "Copy Logs Only" mode: # # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-mohanb/security-integration/guardian-wiki/sdl-azdo-extension/roslyn-analyzers-build-task # -# GOTCHA: This step will clobber any existing build output. It should be run _before_ any build -# steps that perform versioning or signing. +# PROVENANCE: Every statement in this file about how the RoslynAnalyzers task behaves is current as +# of task version v3 (RoslynAnalyzers@3, 3.289.0) and was verified against concrete evidence -- the +# actual pipeline run logs, the task definition (task.json / inputMap.json), the gdn-task-lib task +# source, and the Guardian RoslynAnalyzers CLI binaries (Microsoft.Guardian.RoslynAnalyzers*.dll). +# Re-verify these claims if the task's major version changes. +# +# HOW IT WORKS (integrated analyzers + Copy Logs Only): +# .NET [Roslyn] security analyzers are compiler-integrated: they only run as part of the actual +# csc/vbc compilation. build.proj is an orchestrator -- each package's real compile happens in a +# separate "dotnet build .csproj" that build.proj launches with an task. That is +# the crux: anything the RoslynAnalyzers task appends to an *outer* "dotnet build build.proj" +# command (its auto/manual "re-run the build" modes) is an MSBuild global property, and global +# properties do NOT cross an into a child "dotnet build" process. So an injected analyzer +# would only ever see build.proj (which compiles nothing) and never the real projects, producing +# zero results. That is exactly why earlier auto/manual-mode attempts collected 0 SARIF logs. +# +# Instead we use the task's documented "Copy Logs Only" alternative -- integrate the analyzers into +# the build itself, then have the task only collect the results: +# 1. This step runs its own isolated "dotnet build build.proj -t:Build" with +# EnableAnalyzers=true. build.proj forwards that flag into every leaf "dotnet build" it execs +# (via EnableAnalyzersArgument), and src/Directory.Build.props -- which every product project +# imports -- reacts by enabling the full analyzer set and setting ErrorLog to a per-project +# "*.csproj..sarif" log. src/Directory.Build.targets verifies that each leaf compile +# produced its configured log. Because the analyzers and verification are enabled on the leaf +# projects themselves, they run inside the real compiles regardless of the boundary. +# 2. The RoslynAnalyzers@3 task then runs in Copy Logs Only mode (copyLogsOnly: true) and simply +# collects and sanitizes those *.csproj.*.sarif and *.vbproj.*.sarif logs from +# logRootDirectory for SDL/Guardian compliance. It performs no build and no compiler re-run, +# so none of the msBuildVersion / +# msBuildArchitecture / Visual-Studio-setup concerns apply -- the task never needs MSBuild, +# so it is inherently agnostic to the container's VS/MSBuild version (e.g. MSBuild 18 on the +# ltsc2025/vse2026 image). +# +# EnableAnalyzers and IsolatedBuildPath are independent build.proj properties. We set both here: +# EnableAnalyzers turns analysis on; IsolatedBuildPath keeps this analysis build from disturbing +# real build output (see ISOLATION). +# +# WHAT THE TASK ITSELF INJECTS (v3), AND HOW THIS TEMPLATE COVERS IT: +# In its build-driving modes the task appends five MSBuild properties to the compile command and +# injects the analyzers via user-profile ImportBefore/ImportAfter files. This template enables the +# analyzers on the leaf projects instead, reproducing the effects that matter. Item by item: +# +# | Task injection (v3) | Purpose | How this template covers it | +# |----------------------------------------|-------------------------------------|---------------------------------------------------| +# | /p:Features= | Turns on Roslyn IOperation + | latest-recommended: SDK 18 Roslyn has IOperation | +# | "IOperation,flow-analysis" | dataflow so the taint/crypto | on by default, so these rules run. Add | +# | | security rules (CA3xxx/CA5xxx) run. | flow-analysis to | +# | | | Directory.Build.props if any go missing. | +# | /p:CodeAnalysisRuleSet= | Selects the exact SDL rule IDs + | AnalysisLevel=latest-recommended. This is the | +# | ...Sdl.Recommended.Warning.ruleset | severities; disables non-SDL rules. | SDK's own "recommended" mode, NOT the private SDL | +# | | | ruleset, so complete overlap is not guaranteed; | +# | | | see the CAVEAT in Directory.Build.props. The | +# | | | internal IA* rules need the | +# | | | Microsoft.Internal.Analyzers package, which is | +# | | | Microsoft-internal-only and MUST NOT be added to | +# | | | the public governed feed, so they are not | +# | | | reproduced here. See NOTE ON THE INTERNAL IA* | +# | | | RULES below. | +# | /p:TreatWarningsAsErrors=false | Record every diagnostic instead of | false in the | +# | | failing at the first one. | EnableAnalyzers block of Directory.Build.props. | +# | | | Warning-clean compilation is still enforced by | +# | | | the ordinary build later in each job, which does | +# | | | run with TreatWarningsAsErrors=true. | +# | /p:RunCodeAnalysis=false | Disables legacy *binary* FxCop | Already false by default in SDK-style projects; | +# | | (not the Roslyn analyzers). | we never enable it. | +# | /p:GdnRoslynAnalyzersRunId= | Gates the injected props/targets so | Not needed. We enable analyzers directly on the | +# | | they apply only to this build. | leaf projects, so there is no global injection to | +# | | | gate (see the note below). | +# | ImportBefore *.props / ImportAfter | Adds the analyzer assemblies and | Analyzers: EnableNETAnalyzers + | +# | *.targets under %LOCALAPPDATA%\...\ | sets ErrorLog=.sarif. | latest-recommended (no EnforceCodeStyleInBuild). | +# | MSBuild\Current | | ErrorLog: we set | +# | | | $(MSBuildProjectFullPath)....sarif | +# | | | (SARIF v1 -- NO version=2; see the sanitizer | +# | | | note in Directory.Build.props). | +# +# WHY THE TASK'S OWN INJECTION YIELDS 0 SARIF THROUGH build.proj: the ImportAfter *.targets live in +# the user profile, so they ARE imported by build.proj's inner "dotnet build " execs -- but +# they self-gate on $(GdnRoslynAnalyzersRunId), and that property (like CodeAnalysisRuleSet and +# Features) is passed only on the OUTER "dotnet build build.proj" command and does not cross the +# into the child compiles. So the injected targets no-op in the real compiles. Enabling the +# analyzers on the leaf projects (EnableAnalyzers) removes that gate entirely. +# +# NOTE ON THE INTERNAL IA* RULES: +# The SDL-recommended ruleset also contains internal IA* ("Internal Analyzers") rules that ship in +# the Microsoft.Internal.Analyzers package. That package is Microsoft-internal and confidential: it +# is NOT on nuget.org, and it MUST NOT be added to this repo's governed feed +# (sqlclientdrivers.pkgs.visualstudio.com/public/...), which is PUBLIC-scoped -- doing so would +# leak internal tooling and breach its internal-use license. So the leaf-project analysis above +# (AnalysisLevel=latest-recommended) covers the CA* rules but NOT the IA* rules. +# +# The IA* rules can only be run from the PRIVATE ADO.Net project pipelines, where a Microsoft- +# internal NuGet feed is reachable. NuGet.analysis.config adds that feed using an environment- +# variable placeholder and maps Microsoft.Internal.* exclusively to it. Only these analysis builds +# select that config, allowing Microsoft.Internal.Analyzers to be restored and used without +# changing the normal NuGet.config. +# +# ISOLATION: +# This template is self-contained and safe to run at any point in a job -- before or after a real +# build -- because the analysis build writes its binaries to a separate location and never touches +# the real build output, using the IsolatedBuildPath build.proj property. parameters: # Optional arguments to pass to msbuild to indicate what version of dependencies should be used. @@ -31,41 +129,67 @@ parameters: - SqlClient - SqlServer - # Version revision translated to build.proj's BuildNumber property at this boundary. - - name: revision + # Pre-computed assembly file version, translated to build.proj's FileVersion* property here. + - name: fileVersion type: string # Suffix appended to "PackageVersion" to form the build.proj msbuild property that stamps this - # package's version. build.proj recognizes only two such properties: PackageVersionSqlClient - # (shared by the entire SqlClient family: Logging, Abstractions, SqlClient, Azure, and the AKV - # Provider) and PackageVersionSqlServer (Microsoft.SqlServer.Server). Provided by the caller. - # Examples: 'SqlClient' -> -p:PackageVersionSqlClient=7.1.0-preview3 - # 'SqlServer' -> -p:PackageVersionSqlServer=1.0.0 + # package's version. See build-buildproj-step.yml for the full explanation. - name: versionPropertySuffix type: string # Version to stamp on the package. Combined with versionPropertySuffix to form the msbuild # argument, e.g. -p:PackageVersionSqlClient=7.1.0-preview3. - # Always required — compute up-front via the compute-versions stage. - name: packageVersion type: string steps: - # GOTCHA: If there are any blank lines in msbuildCommandLine, it will consider it "multiple - # arguments" and fail. So, don't split msBuildCommandLine into multiple blocks. - - task: securedevelopmentteam.vss-secure-development-tools.build-task-roslynanalyzers.RoslynAnalyzers@3 - displayName: 'Roslyn Analyzers - build.proj Build${{ parameters.packageShortName }}' + # Step 1: Authenticate to the internal Azure Artifacts feed so the leaf restores can pull + # Microsoft.Internal.Analyzers. NuGetAuthenticate sets up the Azure Artifacts credential provider + # for feeds the build identity can access in this organization. + - task: NuGetAuthenticate@1 + displayName: 'Internal analyzers: authenticate internal feed' + + # Step 2: Isolated analysis build. Compiles the package with EnableAnalyzers=true so that every leaf + # "dotnet build" that build.proj execs turns on the full Roslyn analyzer set and verifies its SARIF. + - task: DotNetCoreCLI@2 + displayName: 'Build for Roslyn analysis - build.proj Build${{ parameters.packageShortName }}' inputs: - msBuildArchitecture: x64 - msBuildCommandLine: >- - msbuild - $(REPO_ROOT)/build.proj + command: build + projects: '$(REPO_ROOT)/build.proj' + arguments: >- -t:Build${{ parameters.packageShortName }} -p:Configuration=Release -p:ReferenceType=Package -p:SkipDependencyPack=true - -p:BuildNumber=${{ parameters.revision }} - -p:PackageVersion${{ parameters.versionPropertySuffix }}=${{ parameters.packageVersion }} + -p:FileVersion${{ parameters.versionPropertySuffix }}="${{ parameters.fileVersion }}" + -p:PackageVersion${{ parameters.versionPropertySuffix }}="${{ parameters.packageVersion }}" + -p:IsolatedBuildPath="$(Agent.TempDirectory)/roslyn" + -p:EnableAnalyzers=true + -p:InternalAnalyzers=true + -p:InternalAnalyzersNugetConfig="$(REPO_ROOT)/NuGet.analysis.config" + -p:InternalAnalyzersVersion=$(InternalAnalyzersVersion) ${{ parameters.dependencyArguments }} - msBuildVersion: 17.0 - setupCommandLinePicker: vs2022 + env: + # dotnet restore expands this environment variable into the NuGet feed URL for + # Microsoft.Internal.Analyzers. + INTERNAL_ANALYZERS_FEED: $(InternalAnalyzersFeed) + + # Step 3: List every SARIF file that the collector will ingest. + - pwsh: | + $sarifFiles = @(Get-ChildItem -Path '$(REPO_ROOT)' -Recurse -File -Include '*.csproj.*.sarif', '*.vbproj.*.sarif' | Sort-Object FullName) + Write-Host "Roslyn collector will ingest $($sarifFiles.Count) SARIF file(s):" + $sarifFiles | ForEach-Object { Write-Host " $($_.FullName)" } + displayName: 'List Roslyn SARIF files for collection' + + # Step 4: Collect the analysis results. In Copy Logs Only mode the task does not build or re-run + # the compiler -- it just gathers and sanitizes the *.csproj.*.sarif and *.vbproj.*.sarif logs + # produced by Step 2 and hands them to Guardian/SDL. + - task: securedevelopmentteam.vss-secure-development-tools.build-task-roslynanalyzers.RoslynAnalyzers@3 + displayName: 'Roslyn Analyzers (collect) - build.proj Build${{ parameters.packageShortName }}' + inputs: + copyLogsOnly: true + # Root to search for the *.csproj.*.sarif and *.vbproj.*.sarif logs. The analysis build wrote + # them next to each project under the repo checkout; the collector globs this directory + # recursively. + logRootDirectory: '$(REPO_ROOT)' diff --git a/eng/pipelines/onebranch/steps/validate-localization-step.yml b/eng/pipelines/onebranch/steps/validate-localization-step.yml new file mode 100644 index 0000000000..8cb590f6d0 --- /dev/null +++ b/eng/pipelines/onebranch/steps/validate-localization-step.yml @@ -0,0 +1,16 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +steps: + - task: PowerShell@2 + displayName: 'Validate localized resources' + inputs: + targetType: filePath + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/pipelines/onebranch/scripts/validate-localization.ps1 + arguments: >- + -ResourcesDirectory "$(Build.SourcesDirectory)/src/Microsoft.Data.SqlClient/src/Resources" + -AllowlistPath "$(Build.SourcesDirectory)/.config/LocalizationValidationAllowlist.json" diff --git a/eng/pipelines/onebranch/steps/validate-packages-step.yml b/eng/pipelines/onebranch/steps/validate-packages-step.yml new file mode 100644 index 0000000000..173df4d7ed --- /dev/null +++ b/eng/pipelines/onebranch/steps/validate-packages-step.yml @@ -0,0 +1,80 @@ +################################################################################# +# Licensed to the .NET Foundation under one or more agreements. # +# The .NET Foundation licenses this file to you under the MIT license. # +# See the LICENSE file in the project root for more information. # +################################################################################# + +# Builds and runs tools/PackageValidator over a directory of produced NuGet packages. +# +# The validator is invoked once for the whole directory rather than once per package, because its +# most valuable checks are cross-package: it confirms that every package in the SqlClient family +# carries the same version and that their inter-package dependency ranges agree. Running it per +# package would silently skip all of those findings. +# +# Two invocations are made over the same inputs. The first writes a machine-readable report and is +# ungated, so the artifact exists even for a failing run. It still fails the step if the validator +# itself errors, because the report it produced could not then be trusted. The second renders the +# human-readable report and applies the gate, so a failed build shows the findings in its own log. + +parameters: + # Directory scanned recursively for .nupkg files. Sibling .snupkg files must sit beside their + # .nupkg for symbol matching to resolve, which is how the build jobs publish them. + - name: packagesPath + type: string + + # Path of the JSON report to write. + - name: reportPath + type: string + + # Expected versions shared by the whole SqlClient family (Logging, Abstractions, SqlClient, + # Azure, AkvProvider), applied as wildcard expectations. Pointing every package at the same + # value is what proves they agree, and also catches every package being consistently wrong. + - name: sqlClientPackageVersion + type: string + + - name: sqlClientFileVersion + type: string + + # Expected versions for the separately-versioned Microsoft.SqlServer.Server, applied as a per-id + # override of the family wildcard. Left empty when SqlServer is not built this run: its package + # is then absent from the drop, and the validator rejects an expectation with an empty value. + - name: sqlServerPackageVersion + type: string + default: '' + + - name: sqlServerFileVersion + type: string + default: '' + + # Finding severities and/or categories that fail the build. Run the validator with --help to + # see the full set of categories. + - name: failOn + type: object + default: + - error + +steps: + - task: DotNetCoreCLI@2 + displayName: 'build.proj - BuildPackageValidator' + inputs: + command: build + projects: '$(REPO_ROOT)/build.proj' + arguments: >- + -t:BuildPackageValidator + -p:Configuration=Release + + - task: PowerShell@2 + displayName: 'Validate NuGet packages' + inputs: + targetType: filePath + pwsh: true + filePath: $(REPO_ROOT)/eng/pipelines/onebranch/scripts/validate-packages.ps1 + arguments: >- + -ValidatorPath "$(REPO_ROOT)/tools/PackageValidator/src/bin/Release/net10.0/PackageValidator.dll" + -PackagesPath "${{ parameters.packagesPath }}" + -ReportPath "${{ parameters.reportPath }}" + -SqlClientPackageVersion "${{ parameters.sqlClientPackageVersion }}" + -SqlClientFileVersion "${{ parameters.sqlClientFileVersion }}" + -SqlServerPackageVersion "${{ parameters.sqlServerPackageVersion }}" + -SqlServerFileVersion "${{ parameters.sqlServerFileVersion }}" + -FailOn "${{ join(',', parameters.failOn) }}" diff --git a/eng/pipelines/onebranch/variables/onebranch-variables.yml b/eng/pipelines/onebranch/variables/onebranch-variables.yml index fb671961cc..b6d4ce7d5c 100644 --- a/eng/pipelines/onebranch/variables/onebranch-variables.yml +++ b/eng/pipelines/onebranch/variables/onebranch-variables.yml @@ -32,6 +32,15 @@ variables: # SymbolsUploadAccount - group: 'symbols-variables-v3' + # These variables point the SDL Roslyn analysis step at the internal Microsoft.Internal.Analyzers + # package (the "IA*" rules). The package is Microsoft-internal and confidential, so the feed URL + # and pinned version live in this ADO.Net-project variable group rather than in the repo. Consumed + # by eng/pipelines/onebranch/steps/roslyn-analyzers-buildproj-step.yml. + # + # InternalAnalyzersFeed + # InternalAnalyzersVersion + - group: 'internal-analyzers-variables-v1' + # Well-Known Variables ################################################### # Directory where downloaded pipeline artifacts (NuGet packages from earlier @@ -54,16 +63,13 @@ variables: - name: Packaging.EnableSBOMSigning value: true - # Keep this as a runtime variable reference in globalSdl.apiscan.versionNumber. Passing the - # quoted value directly through an Azure template expression perturbs '6.10' to the number 6.1. - - name: ApiScanSoftwareVersion - value: '6.10' - # OneBranch supplies a variety of container images we must use for our jobs. + # + # https://eng.ms/docs/products/onebranch/infrastructureandimages/containerimages/containerimages # Windows jobs use this image. - name: WindowsContainerImage - value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + value: onebranch.azurecr.io/windows/ltsc2025/vse2026:latest # Linux jobs use this image. - name: LinuxContainerImage diff --git a/eng/pipelines/onebranch/variables/package-variables.yml b/eng/pipelines/onebranch/variables/package-variables.yml index 62afca6605..65fd0867d0 100644 --- a/eng/pipelines/onebranch/variables/package-variables.yml +++ b/eng/pipelines/onebranch/variables/package-variables.yml @@ -7,23 +7,6 @@ # This file contains variables that relate to the various packages that are produced via the # OneBranch official/non-official pipelines. They are grouped by the packages they represent. # -# VERSION STRATEGY -# ================ -# Package versions are computed by Versions.props using one input: -# - BuildNumber ($(Build.BuildNumber), provided by ADO) -# -# The SqlClient *family* (Internal.Logging, Extensions.Abstractions, Microsoft.Data.SqlClient, -# Extensions.Azure, and the AlwaysEncrypted.AzureKeyVaultProvider) shares a single version defined -# by SqlClientNextVersion in src/Microsoft.Data.SqlClient/Versions.props. Microsoft.SqlServer.Server -# is versioned separately by SqlServerNextVersion. -# -# The *NextVersion properties define the base version: -# - On main: X.Y.Z-preview1 → official produces "X.Y.Z-preview1" -# - On release/X.Y: X.Y.Z → official produces "X.Y.Z" -# -# No per-package version variables are needed here — Versions.props is the single source. -# Downstream stages obtain the computed versions via the compute-versions stage. -# # ARTIFACT NAMING # =============== # OneBranch automatically publishes pipeline artifacts from all jobs named as: diff --git a/eng/pipelines/perf/README.md b/eng/pipelines/perf/README.md index be925f5431..2e4c62185b 100644 --- a/eng/pipelines/perf/README.md +++ b/eng/pipelines/perf/README.md @@ -12,7 +12,8 @@ database. | ---- | ------- | | `sqlclient-perf-pipeline.yml` | The main (manual/nightly) pipeline. Extends `v1/Perf.Test.Job.yml@PerfTemplates`. Baseline = released NuGet package; ingests into Kusto. | | `sqlclient-perf-pr-pipeline.yml` | PR pipeline. Same template, same scripts, same options; baseline = **`main` branch source**; **no Kusto ingestion**. | -| `scripts/run-perf-tests.sh` | Linux on-VM entry point: install SDK, create DB, run benchmarks (interleaved or sequential), compare. Baseline is either a released package (`--baseline-version`) or another git ref's source (`--baseline-source-ref`). | +| `sqlclient-perf-experiment.yml` | Experiment pipeline. Same template, same scripts; both passes build the **same source** and differ only in one runner-config switch; **no Kusto ingestion**. | +| `scripts/run-perf-tests.sh` | Linux on-VM entry point: install SDK, create DB, run benchmarks (interleaved or sequential), compare. Baseline is a released package (`--baseline-version`), another git ref's source (`--baseline-source-ref`), or the same source with one runner-config switch flipped off (`--switch-under-test`). | | `scripts/run-perf-tests.ps1` | Windows equivalent (ProcessorAffinity instead of `taskset`). | | `scripts/interleave_perf.py` | Interleaved + best-of-N orchestrator: runs each unit baseline↔candidate back-to-back and confirms regressions across N passes. | | `scripts/compare_perf.py` | Compares baseline vs current BenchmarkDotNet JSON → delta (md + json). Reused by the orchestrator. | @@ -151,6 +152,122 @@ the comparison (and one removed by the PR as `removed`) instead of failing the r share the single generated runner config (`RUNNER_CONFIG` / `DATATYPES_CONFIG` env vars), so connection string and behaviour flags are identical on both sides. +## Experiment pipeline (`sqlclient-perf-experiment.yml`) + +The three perf pipelines are the same benchmarks, template and scripts pointed at three different +questions. Two of them vary the **source** under measurement; the third varies the **config**: + +| Question | Pipeline | Baseline | Current | +| --- | --- | --- | --- | +| Has this branch regressed against a released package? | `sqlclient-perf-pipeline.yml` | released NuGet package | queued branch | +| Does my PR regress the branch it merges into? | `sqlclient-perf-pr-pipeline.yml` | `main` source | queued branch | +| What does this switch cost or buy? | `sqlclient-perf-experiment.yml` | queued branch, switch **off** | queued branch, switch **on** | + +`sqlclient-perf-experiment.yml` picks one runner-config switch via the `switchUnderTest` +queue-time parameter (`UseConnectionPoolV2`, `UseOptimizedAsyncBehaviour` or +`UseManagedSniOnWindows`) and runs the baseline pass with it `false` and the current pass with it +`true`. Both passes measure the **same commit** — the branch the run is queued on — so queue it on a +PR branch to ask "what does this switch do to my change?", or on `main` to ask "what does it do to +`main`?". + +Two passes are required because these are `AppContext` switches latched process-wide (for example +`UseConnectionPoolV2` is read and cached the first time a connection pool is created), so they cannot +be toggled between benchmarks within a single process. The pipeline wires that into the existing +comparison machinery — interleaved best-of-N or sequential, via `benchmarkRunMode` — instead of two +ad hoc manual runs. + +Switch-pipeline parameters that differ from the tables above: + +| Parameter | Default | Description | +| --------- | ------- | ----------- | +| `switchUnderTest` | `UseConnectionPoolV2` | The single switch to A/B. Baseline forces it `false`, current forces it `true`. | +| `failIfSwitchSlower` | `false` | Maps onto the scripts' `--fail-on-regression` gate, but means something different here: "switch on is slower" is usually the *result* you queued the run to measure, not a defect. Enable it only when asserting the switch must not be a slowdown (e.g. before flipping its default). | +| `testTimeoutMinutes` | `180` | Same as the main pipeline, not the PR pipeline's `210`: both sides are the same source, so only **one** driver build is needed. | + +The pipeline deliberately does **not** expose `baselineVersion` / `baselineSourceRef` (the run +scripts reject combining those with `--switch-under-test`, since a simultaneous source change would +make the delta unattributable), nor the `useManagedSniOnWindows` / `useOptimizedAsyncBehaviour` / +`useConnectionPoolV2` flags. Every switch except the one under test stays at its checked-in +`runnerconfig.jsonc` value, so the measured difference is attributable to exactly one variable. + +### Why these runs are never ingested into Kusto + +This mode has its own pipeline file, rather than being a flag on the other two, specifically so that +"never ingested" is structural rather than a conditional someone can flip. Ingesting a switch +experiment would corrupt the perf database three ways: + +* **`DerivedRunId` collision.** The ID is `driver|commit|pipelineRunId`. The other two pipelines keep + their two rows distinct because the baseline row carries a *different* commit (`v7.0.2`, or the + baseline ref's sha). Here both passes are the same commit in the same pipeline run, so both rows + would derive the same ID. +* **`PerfRun.Config` is stamped once per run.** `translate_results_to_kusto.sh` builds one + `--config-override` set from the queue-time `CFG_*` values and reuses it for both the baseline and + current rows. That is correct when the config genuinely is shared, but it means the two rows could + not record the differing switch values that are the entire point of the experiment. +* **Trend pollution.** No field marks a row as an experiment — `RunType` is already + `Sequential`/`Interweaved` — so the switch-on pass would be indistinguishable from an ordinary + measurement of the branch and would distort the very trends the other two pipelines exist to + protect. + +The comparison report and the raw BenchmarkDotNet artifacts are published as usual, and the build is +tagged `Switch ` so experiments are identifiable in the ADO build list. + +### Designing benchmarks for switch experiments + +A switch experiment flips behaviour on purpose, so a benchmark that measures that behaviour will +report a regression even when the change is working. `UseConnectionPoolV2` is the motivating example: +`ChannelDbConnectionPool` opens physical connections concurrently, where `WaitHandleDbConnectionPool` +serialises growth behind a `Semaphore(1, 1)`. `ConnectionPoolStressRunner` used to call +`ClearAllPools()` in `[IterationCleanup]`, so every iteration was a cold-start burst in which the +extra parallel opens had nothing to amortise against, and the runner reported the trade-off as a loss +because that was the only thing it could measure. + +That is a benchmark defect rather than something to explain away, so the runner now pre-warms the +pool to full capacity in `[GlobalSetup]` and no longer clears it between iterations. Establishing a +connection costs milliseconds while a pooled checkout costs microseconds, so any creation left in the +measured body swamps the pool cost the runner exists to measure. Keep that separation in mind when +adding a pool benchmark: warm the pool first unless connection establishment is precisely the thing +under test. + +The pipeline has no way to mark a result as acceptable, and deliberately so: a mute is only as good +as the reasoning behind it, and that reasoning belongs in the pull request where a reviewer can +challenge it. Prefer instead to fix the benchmark, or add one that measures the intended behaviour +directly. `ConnectionPoolRampRunner` was added for exactly this reason: it keeps the cold pool but +makes every caller *hold* its connection until all of them have connected, so the pool genuinely needs +N physical connections and the only variable left is how fast it can open them. That rewards +concurrent creation instead of penalising it, and it isolates cold start now that the stress runner no +longer conflates it with checkout cost. + +The same principle applies to how a benchmark schedules its workers. A sync `Open()` that has to +wait blocks whichever thread it runs on, so a pool whose waiter wake-up needs a queued continuation +stalls when every threadpool thread is already blocked; the wake-up waits on thread injection. On +the TFMs the perf project builds (net8.0-net10.0) the runtime is told about cooperative blocking and +compensates quickly, so that stall is tens to a few hundred milliseconds, and that is the only +expectation these benchmarks validate. On net462 the `Task` wait never notifies the pool, so the +wake-up falls to starvation detection and hill climbing and is materially slower; the pool carries no +framework guards and the driver still ships net462, so that path is live but unmeasured here. +Threadpool threads are the realistic case, because sync database +calls in ASP.NET run on them, and they are the only configuration in which that stall is visible. +Benchmarks therefore keep threadpool threads as the default and add dedicated-thread variants +alongside rather than instead: + +- `ConnectionPoolContentionRunner.SteadyStateOpenQueryCloseDedicatedThreads` runs the existing + workload on dedicated threads. A regression in both variants points at the pool; a regression in + only the threadpool variant points at the waiter wake path. +- `ConnectionPoolThreadPoolPressureRunner` pins the threadpool floor via `[Params]`, below the worker + count (starved) and above it (control), so the effect is reproducible instead of depending on + hill-climbing timing. + +This failure mode is tail latency, not a shifted median, so compare distributions. Aggregating with +a per-configuration minimum hides it completely. + +Note what those benchmarks are for. Saturating the thread pool with blocked synchronous calls is an +application configuration problem, not a pool defect: an application should keep its parallelism +below the thread pool's worker count so newly queued work still runs promptly, and pre-warming the +thread pool is the application's responsibility rather than the driver's. These benchmarks exist to +characterise where that boundary sits and to catch it moving, so a delta here is a prompt to check +the boundary has not shifted rather than a bug to fix. + ## Two-pass build model The `PerformanceTests` project references Microsoft.Data.SqlClient two ways, selected by MSBuild: @@ -162,6 +279,9 @@ The `PerformanceTests` project references Microsoft.Data.SqlClient two ways, sel - **Baseline (source)**: no reference switching at all — the baseline ref's own copy of the perf project is built from `../sqlclient-perf-baseline-src`, keeping its default `ProjectReference` to that ref's driver source. Used by the PR pipeline. +- **Baseline (switch experiment)**: no second build at all — `--switch-under-test` measures one + source tree twice, so the scripts build the `current` variant once and point both passes at it, + differing only in the runner config each pass is handed. Used by the experiment pipeline. The VM's `NuGet.config` exposes only the governed feed, and CPM rejects multiple unmapped sources (`NU1507`). The baseline pass therefore restores through a **dedicated single-source config** @@ -210,12 +330,29 @@ supplies the isolated dedicated host, the tuned SQL instance, and the disjoint c | Fail loud | Preflight `SELECT 1` before any pass, **and** a post-pass guard that fails the run if a pass produced **zero** benchmark results — so an empty comparison can never be reported green. | | Warm-up | Touches the target DB in the preflight to warm the buffer pool / plan cache before the first measured benchmark. | | Allocator tuning (Linux) | Exports `MALLOC_MMAP_THRESHOLD_=128MiB` and `MALLOC_TRIM_THRESHOLD_=-1` so large-buffer benches (`LargeDataRead`, `SqlBulkCopy`) stop re-`mmap`ing per iteration. | -| Network tuning (Linux) | Best-effort `sysctl` to widen the ephemeral port range and enable `tcp_tw_reuse` for churn benches (`ConnectionPoolStress`, `ParallelAsyncConnection`). Never fails the run. | +| Network tuning (Linux) | Best-effort `sysctl` to widen the ephemeral port range and enable `tcp_tw_reuse` for churn benches (`ConnectionPoolStress`, `ConnectionPoolRamp`, `ConnectionPoolThreadPoolPressure`, `ParallelAsyncConnection`). Never fails the run. | | Diagnostics | Writes `results/diagnostics/`: SQL instance config (MAXDOP, memory, affinity, tempdb files, `@@VERSION`), host CPU topology, and per-pass CPU-clock/thermal telemetry (before/after each pass). | | Regression gate | `failOnRegression` threads `--fail-on-regression`; only a **candidate-slower** delta past the threshold fails, and in interleaved mode only after best-of-N confirmation. Default off. | | Interleaving | In `interleaved` mode the harness runs **one benchmark unit at a time, baseline then candidate back-to-back**, so both sides see the same host state (see below). | | Best-of-N confirmation | A unit flagged in the first interleaved pass is re-run `confirmationRuns` times; a regression is **confirmed** only on a strict majority. Unconfirmed flags are reported but never fail the gate. | +### Windows physical opens + +The Windows connection probe measured raw TCP at about 0.06 ms but a physical +`SqlConnection.Open()` at about 158 ms when the runner used the `sa` SQL login, with the same result +from native and managed SNI. Changing TLS and TCP acknowledgment settings did not change that +latency. The Windows image runs SQL Server 2025, which verifies SQL-login passwords with 100,000 +PBKDF2 iterations; Microsoft documents the resulting login-performance impact. + +The benchmark process runs through a public-key SSH token, which cannot delegate Windows credentials +to the VM's private network address. When the injected SQL endpoint resolves to a local interface, +the Windows harness connects to the same SQL Server 2025 instance over local TCP loopback with +integrated authentication. It grants the ephemeral VM identity `db_owner` in the perf database; `sa` +remains unchanged and is used only for one-time setup. External SQL endpoints continue to use the +injected address and SQL authentication. This stays on SQL Server 2025's supported authentication +path while preventing password hashing from dominating local tests that intentionally create +physical connections. + ### Interleaving + best-of-N (run model) `benchmarkRunMode` selects how the two variants are measured: @@ -323,6 +460,18 @@ translated NDJSON as the `perf-kusto-payloads` artifact for manual/backfill inge 3. After the run, review the **run summary** (comparison, labelled `@`) and the `perf-results` artifact. The build is tagged **`Baseline `**. +### Running the experiment pipeline + +1. Open the **experiment** performance test pipeline (`sqlclient-perf-experiment.yml`) in Azure + DevOps and select **Run pipeline**. +2. Choose the branch whose behaviour you want to measure — `main` to characterise the switch on its + own, or a PR branch to characterise it against that change — and pick `switchUnderTest`. There is + no baseline selector: the baseline *is* this branch with the switch off. No Kusto configuration is + involved; these results are never ingested (see [Why these runs are never ingested into + Kusto](#why-these-runs-are-never-ingested-into-kusto)). +3. After the run, review the **run summary** (comparison, labelled `=false`) and the + `perf-results` artifact. The build is tagged **`Switch `**. + ## Troubleshooting | Symptom | Likely cause / fix | @@ -331,6 +480,8 @@ translated NDJSON as the `perf-kusto-payloads` artifact for manual/backfill inge | Baseline restore fails to find MDS | `baselineVersion` isn't a published NuGet.org version, or the VM has no outbound access to `api.nuget.org`. | | Baseline pass fails to compile: `CS1061 ... does not contain a definition for ` | A benchmark calls an MDS API newer than `baselineVersion`. Guard it with the `MDS_GE_` constants and add an older fallback — see [Benchmarks must compile against the oldest baseline](#benchmarks-must-compile-against-the-oldest-baseline). | | No comparison / summary | The baseline pass was skipped (empty `baselineVersion` / `baselineSourceRef`) or one pass produced no `*-report-full.json`. | +| `--switch-under-test is mutually exclusive with --baseline-version and --baseline-source-ref` | A switch experiment was combined with a source baseline. The experiment pipeline never does this; if you are invoking the scripts directly, clear the baseline selector — varying source and config at once makes the delta unattributable. | +| Switch experiment shows a ~0% delta everywhere | Expected for benchmarks the switch does not touch. If *every* benchmark is flat, check the run log's `Switch A/B` line actually names the switch, and that the switch is one the driver reads at startup via the runner config. | | Baseline source ref not found (PR pipeline) | `baselineSourceRef` is not a branch on `origin`, and the fallback `git clone --branch ` of `baselineRepoUrl` also failed (ref does not exist there, or the VM has no outbound access to the remote). The reason git gave is echoed into the build log and saved to `/diagnostics/git-*.log`. | | `Fetching baseline ref ... from the checkout's origin` is the last line, then the job stalls | Should no longer happen. The checkout is copied to the VM without credentials (ADO's checkout task defaults to `persistCredentials: false`), so a fetch from an authenticated `origin` used to sit on a `Username for ...` prompt forever. All network git calls now run with `GIT_TERMINAL_PROMPT=0`, no credential helper, stdin closed, and a `GIT_NET_TIMEOUT_SECS` (default 300s) hard timeout, so this fails in under a second and falls back to cloning `baselineRepoUrl`. A fetch failure here is expected and harmless on ADO. | | Ingestion step skipped | `enableKustoIngestion` is `false`, or `KustoClusterUri`, `KustoDatabase` or `KustoServiceConnection` (from `ADX Cluster Variables`) is empty (expected until the cluster is provisioned). | diff --git a/eng/pipelines/perf/scripts/interleave_perf.py b/eng/pipelines/perf/scripts/interleave_perf.py index 167002fb8f..6522adced1 100644 --- a/eng/pipelines/perf/scripts/interleave_perf.py +++ b/eng/pipelines/perf/scripts/interleave_perf.py @@ -101,17 +101,22 @@ def apply_affinity(proc, cpus): # -------------------------------------------------------------------------------------------------- # Running one unit and collecting its artifacts. # -------------------------------------------------------------------------------------------------- -def run_unit_process(exe_dir, assembly, unit, cwd, cpus, log_path): +def run_unit_process(exe_dir, assembly, unit, cwd, cpus, log_path, env_overrides=None): """Run one benchmark *unit* from the build at *exe_dir* in *cwd*. Returns the subprocess return code. Kept as a small seam so tests can substitute - a fake runner. + a fake runner. *env_overrides*, when given, is applied on top of the inherited + environment (e.g. a per-variant RUNNER_CONFIG so baseline and current can run with + different SqlClient behaviour flags, such as comparing the legacy vs new connection + pool from the SAME build). """ os.makedirs(cwd, exist_ok=True) cmd = ["dotnet", os.path.join(exe_dir, assembly)] env = dict(os.environ) env["PERF_BENCHMARK"] = unit env.pop("PERF_LIST_BENCHMARKS", None) + if env_overrides: + env.update(env_overrides) with open(log_path, "w", encoding="utf-8") as log: proc = subprocess.Popen(cmd, cwd=cwd, env=env, stdout=log, @@ -154,12 +159,18 @@ def collect_results(cwd, dest): class Runner: """Holds the invariant run parameters and performs interleaved unit passes.""" - def __init__(self, baseline_dir, current_dir, assembly, work_dir, cpus): + def __init__(self, baseline_dir, current_dir, assembly, work_dir, cpus, + baseline_runner_config=None, current_runner_config=None): self.baseline_dir = baseline_dir self.current_dir = current_dir self.assembly = assembly self.work_dir = work_dir self.cpus = cpus + # Optional per-variant RUNNER_CONFIG override (e.g. --switch-under-test needs baseline + # and current to run with different SqlClient behaviour flags even though they share the + # same build). None means "no override" -> both variants use the ambient RUNNER_CONFIG. + self.baseline_runner_config = baseline_runner_config + self.current_runner_config = current_runner_config def list_units(self): cmd = ["dotnet", os.path.join(self.current_dir, self.assembly)] @@ -175,7 +186,11 @@ def _run_one(self, variant, exe_dir, unit, rep, agg_dir): shutil.rmtree(cwd) os.makedirs(cwd, exist_ok=True) log_path = os.path.join(cwd, "run.log") - rc = run_unit_process(exe_dir, self.assembly, unit, cwd, self.cpus, log_path) + runner_config = (self.baseline_runner_config if variant == "baseline" + else self.current_runner_config) + env_overrides = {"RUNNER_CONFIG": runner_config} if runner_config else None + rc = run_unit_process(exe_dir, self.assembly, unit, cwd, self.cpus, log_path, + env_overrides=env_overrides) if rc != 0: _tail(log_path) raise RuntimeError(f"benchmark unit '{unit}' ({variant}, rep {rep}) failed (exit {rc}).") @@ -353,6 +368,14 @@ def main(argv=None): parser.add_argument("--reps", type=int, default=3, help="Total interleaved passes for a flagged unit (best-of-N). 1 disables confirmation.") parser.add_argument("--baseline-version", default="baseline") + parser.add_argument("--baseline-runner-config", default=None, + help="Override RUNNER_CONFIG for baseline-variant subprocesses only " + "(e.g. to force a different SqlClient behaviour flag for the " + "baseline, such as comparing the legacy vs new connection pool " + "from the same build). Omit to use the ambient RUNNER_CONFIG for " + "both variants (default behaviour).") + parser.add_argument("--current-runner-config", default=None, + help="Override RUNNER_CONFIG for current-variant subprocesses only.") parser.add_argument("--client-cpus", default=os.environ.get("PERF_CLIENT_CPUS", ""), help="CPU set to pin the benchmark client to, e.g. '16-31'.") parser.add_argument("--fail-on-regression", action="store_true", @@ -376,7 +399,9 @@ def main(argv=None): runner = Runner(os.path.abspath(args.baseline_exe_dir), os.path.abspath(args.current_exe_dir), - args.assembly, work_dir, cpus) + args.assembly, work_dir, cpus, + baseline_runner_config=args.baseline_runner_config, + current_runner_config=args.current_runner_config) units = runner.list_units() if not units: diff --git a/eng/pipelines/perf/scripts/run-perf-tests.ps1 b/eng/pipelines/perf/scripts/run-perf-tests.ps1 index c91239eb80..2b3f9bed40 100644 --- a/eng/pipelines/perf/scripts/run-perf-tests.ps1 +++ b/eng/pipelines/perf/scripts/run-perf-tests.ps1 @@ -54,7 +54,17 @@ param( [ValidateSet("", "true", "false")] [string]$UseOptimizedAsyncBehaviour = "", [ValidateSet("", "true", "false")] - [string]$UseConnectionPoolV2 = "" + [string]$UseConnectionPoolV2 = "", + # Alternative to -BaselineVersion/-BaselineSourceRef: an A/B experiment on ONE runner-config + # switch. Both passes build the SAME source; only the named switch differs (baseline=false, + # current=true), which is the only way to compare a switch whose value is latched process-wide + # (e.g. UseConnectionPoolV2 is read and cached the first time a pool is created). Mutually + # exclusive with the other two baseline selectors, and overrides the matching -Use* flag (which + # would otherwise be ambiguous: one value cannot describe two passes). ValidateSet restricts it + # to switches this script knows how to stamp, so a typo fails fast at binding time instead of + # silently writing an inert key and reporting a meaningless zero-delta comparison. + [ValidateSet("", "UseConnectionPoolV2", "UseOptimizedAsyncBehaviour", "UseManagedSniOnWindows")] + [string]$SwitchUnderTest = "" ) $ErrorActionPreference = "Stop" @@ -128,6 +138,7 @@ Write-Host " Results dir : $ResultsDir" Write-Host " Run mode : $RunMode (confirmation runs: $ConfirmationRuns)" Write-Host " Baseline ver : $(if ($BaselineVersion) { $BaselineVersion } else { '' })" Write-Host " Baseline ref : $(if ($BaselineSourceRef) { $BaselineSourceRef } else { '' })" +Write-Host " Switch A/B : $(if ($SwitchUnderTest) { "$SwitchUnderTest (baseline=false vs current=true)" } else { '' })" Write-Host " SQL_SERVER : $SqlServer" Write-Host " PERF_CLIENT_CPUS: $($env:PERF_CLIENT_CPUS)" Write-Host " PERF_SQL_CPUS : $($env:PERF_SQL_CPUS)" @@ -141,6 +152,32 @@ if (-not (Test-Path $PerfProject)) { if ((-not [string]::IsNullOrEmpty($BaselineVersion)) -and (-not [string]::IsNullOrEmpty($BaselineSourceRef))) { throw "-BaselineVersion and -BaselineSourceRef are mutually exclusive." } +if ((-not [string]::IsNullOrEmpty($SwitchUnderTest)) -and ((-not [string]::IsNullOrEmpty($BaselineVersion)) -or (-not [string]::IsNullOrEmpty($BaselineSourceRef)))) { + throw "-SwitchUnderTest is mutually exclusive with -BaselineVersion and -BaselineSourceRef: it compares the SAME source build with one switch flipped, so mixing in a source change would make the delta unattributable." +} +# UseManagedSniOnWindows only selects an implementation on Windows. PowerShell Core also runs on +# Linux, so check the host rather than assume this script implies Windows: off-Windows both passes +# use managed SNI and the experiment reports a ~0% delta that reads as "the switch is free". +# $IsWindows is undefined on Windows PowerShell 5.1, which is itself Windows-only, so null is Windows. +if ($SwitchUnderTest -eq "UseManagedSniOnWindows") { + $onWindows = if ($null -eq (Get-Variable -Name IsWindows -ErrorAction SilentlyContinue)) { $true } else { $IsWindows } + if (-not $onWindows) { + throw "-SwitchUnderTest UseManagedSniOnWindows is Windows-only; elsewhere managed SNI is always used, so baseline and current would be identical. Re-run this experiment on the Windows platform." + } +} +# -SwitchUnderTest forces its switch explicitly for each pass (baseline=false, current=true), so a +# separately-supplied -Use* flag for that SAME switch would be silently overridden; warn rather than +# let that go unnoticed. Other -Use* flags still apply normally to both passes. +$conflictingFlag = "" +$conflictingFlagValue = switch ($SwitchUnderTest) { + "UseConnectionPoolV2" { $conflictingFlag = "-UseConnectionPoolV2"; $UseConnectionPoolV2 } + "UseOptimizedAsyncBehaviour" { $conflictingFlag = "-UseOptimizedAsyncBehaviour"; $UseOptimizedAsyncBehaviour } + "UseManagedSniOnWindows" { $conflictingFlag = "-UseManagedSniOnWindows"; $UseManagedSniOnWindows } + default { "" } +} +if (-not [string]::IsNullOrEmpty($conflictingFlagValue)) { + Write-Warning "$conflictingFlag=$conflictingFlagValue is ignored when -SwitchUnderTest is $SwitchUnderTest (baseline forces false, current forces true)." +} if ([string]::IsNullOrEmpty($SqlPassword)) { throw "SQL_PASSWORD environment variable is not set (expected from the perf template)." } @@ -302,20 +339,11 @@ $env:RUNNER_CONFIG = $RunnerConfig # It needs no per-run modification, so point the env var at the checked-in file directly. $env:DATATYPES_CONFIG = Join-Path $PerfDir "datatypes.json" -$srcConfig = Join-Path $PerfDir "runnerconfig.jsonc" -$rawConfig = Get-Content $srcConfig -Raw -# Strip // line comments so ConvertFrom-Json accepts the .jsonc content. -$rawConfig = ($rawConfig -split "`n" | ForEach-Object { $_ -replace '(?m)^\s*//.*$', '' }) -join "`n" -$cfg = ConvertFrom-Json $rawConfig - # SqlClient connection-string values may be wrapped in double quotes; doubling any embedded double # quote lets a password containing ';', '=', spaces or single quotes be parsed as a single literal # value instead of corrupting the connection string. $escapedPassword = '"' + ($SqlPassword -replace '"', '""') + '"' -$cfg.ConnectionString = "Server=tcp:$SqlServer,1433;User ID=sa;Password=$escapedPassword;Initial Catalog=$DbName;TrustServerCertificate=True;Encrypt=False;" -# Apply the optional SqlClient behaviour overrides supplied by the pipeline. An empty value leaves -# the checked-in default untouched; otherwise the flag is forced to the requested boolean so the -# benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. + function Set-CfgBool { param($Config, [string]$Name, [string]$Value) if (-not [string]::IsNullOrEmpty($Value)) { @@ -324,11 +352,51 @@ function Set-CfgBool { else { $Config | Add-Member -NotePropertyName $Name -NotePropertyValue $b } } } -Set-CfgBool $cfg "UseManagedSniOnWindows" $UseManagedSniOnWindows -Set-CfgBool $cfg "UseOptimizedAsyncBehaviour" $UseOptimizedAsyncBehaviour -Set-CfgBool $cfg "UseConnectionPoolV2" $UseConnectionPoolV2 -$cfg | ConvertTo-Json -Depth 10 | Set-Content -Path $RunnerConfig -Encoding UTF8 -Write-Host "Wrote runner config to $RunnerConfig (Server=tcp:$SqlServer,1433; Initial Catalog=$DbName)" + +# Write-RunnerConfig [switchName] [switchValue] +# Writes one runner config (checked-in runnerconfig.jsonc + injected connection string + behaviour +# overrides) to . When is given, that config key is forced to +# ("true"/"false") regardless of the corresponding -Use* parameter -- used by -SwitchUnderTest, which +# needs a different value for the same switch in each pass. With no switch name the config is built +# purely from the -Use* parameters, exactly as before. +function Write-RunnerConfig { + param([string]$Dst, [string]$SwitchName = "", [string]$SwitchValue = "") + $srcConfig = Join-Path $PerfDir "runnerconfig.jsonc" + $rawConfig = Get-Content $srcConfig -Raw + # Strip // line comments so ConvertFrom-Json accepts the .jsonc content. + $rawConfig = ($rawConfig -split "`n" | ForEach-Object { $_ -replace '(?m)^\s*//.*$', '' }) -join "`n" + $cfg = ConvertFrom-Json $rawConfig + + $cfg.ConnectionString = "Server=tcp:$SqlServer,1433;User ID=sa;Password=$escapedPassword;Initial Catalog=$DbName;TrustServerCertificate=True;Encrypt=False;" + # Apply the optional SqlClient behaviour overrides supplied by the pipeline. An empty value + # leaves the checked-in default untouched; otherwise the flag is forced to the requested boolean + # so the benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. The + # switch-under-test override (when this config names one) takes precedence over the matching + # -Use* parameter, so a single checked-in template can be stamped out per pass with that one + # switch flipped and everything else identical. + $sniValue = if ($SwitchName -eq "UseManagedSniOnWindows") { $SwitchValue } else { $UseManagedSniOnWindows } + $asyncValue = if ($SwitchName -eq "UseOptimizedAsyncBehaviour") { $SwitchValue } else { $UseOptimizedAsyncBehaviour } + $poolValue = if ($SwitchName -eq "UseConnectionPoolV2") { $SwitchValue } else { $UseConnectionPoolV2 } + Set-CfgBool $cfg "UseManagedSniOnWindows" $sniValue + Set-CfgBool $cfg "UseOptimizedAsyncBehaviour" $asyncValue + Set-CfgBool $cfg "UseConnectionPoolV2" $poolValue + $cfg | ConvertTo-Json -Depth 10 | Set-Content -Path $Dst -Encoding UTF8 + Write-Host "Wrote runner config to $Dst (Server=tcp:$SqlServer,1433; Initial Catalog=$DbName)" +} + +Write-RunnerConfig -Dst $RunnerConfig + +# -SwitchUnderTest needs two DIFFERENT runner configs (baseline runs the switch off, current runs it +# on), so stamp out two more copies here alongside the shared one above. Everything else in them is +# identical, so any measured delta is attributable to the switch alone. +$BaselineRunnerConfig = "" +$CurrentRunnerConfig = "" +if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { + $BaselineRunnerConfig = Join-Path $RepoRoot "perf-runnerconfig-baseline.json" + $CurrentRunnerConfig = Join-Path $RepoRoot "perf-runnerconfig-current.json" + Write-RunnerConfig -Dst $BaselineRunnerConfig -SwitchName $SwitchUnderTest -SwitchValue "false" + Write-RunnerConfig -Dst $CurrentRunnerConfig -SwitchName $SwitchUnderTest -SwitchValue "true" +} #################################################################################################### # 4 & 5. Run the benchmarks, pinned to the reserved client CPU set. @@ -688,6 +756,11 @@ if (-not [string]::IsNullOrEmpty($BaselineVersion)) { $baselineSource = Initialize-BaselineSource -Ref $BaselineSourceRef $BaselineLabel = $baselineSource.Label $BaselineProject = $baselineSource.Project +} elseif (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { + # Switch A/B: SAME source/project for both passes ($BaselineProject/$BaselineBuildArgs are + # already the candidate's, set above), so only the runner config differs (see + # $BaselineRunnerConfig/$CurrentRunnerConfig written above: the named switch off vs on). + $BaselineLabel = "$SwitchUnderTest=false" } # Record the resolved baseline label (for a source baseline this is '@') in the results @@ -704,8 +777,17 @@ if ((-not [string]::IsNullOrEmpty($BaselineLabel)) -and ($RunMode -eq "interleav # orchestrator run one unit at a time (baseline then candidate) and confirm any flagged # regression across N passes before it counts toward the gate. #################################################################################################### - $baselineExeDir = Build-Variant "baseline" $BaselineProject $BaselineBuildArgs - $currentExeDir = Build-Variant "current" $PerfProject @() + if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { + # Switch A/B measures one build against itself with a switch flipped, so building the same + # project twice would just burn several minutes producing identical bits. Build once and + # point both variants at it; the orchestrator runs each variant in its own working directory + # (rep//), so a shared exe dir cannot cross-contaminate their artifacts. + $currentExeDir = Build-Variant "current" $PerfProject @() + $baselineExeDir = $currentExeDir + } else { + $baselineExeDir = Build-Variant "baseline" $BaselineProject $BaselineBuildArgs + $currentExeDir = Build-Variant "current" $PerfProject @() + } $interleaveArgs = @( "--baseline-exe-dir", $baselineExeDir, @@ -717,6 +799,12 @@ if ((-not [string]::IsNullOrEmpty($BaselineLabel)) -and ($RunMode -eq "interleav "--baseline-version", $BaselineLabel, "--client-cpus", "$($env:PERF_CLIENT_CPUS)" ) + # -SwitchUnderTest: baseline and current subprocesses need DIFFERENT RUNNER_CONFIG values (the + # switch off vs on), even though both are otherwise the same build/env; every other baseline + # flavour keeps sharing the single ambient RUNNER_CONFIG set above. + if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { + $interleaveArgs += @("--baseline-runner-config", $BaselineRunnerConfig, "--current-runner-config", $CurrentRunnerConfig) + } if ($FailOnRegression) { Write-Host "Regression gate ENABLED: a CONFIRMED candidate-slower regression (> $RegressionThreshold%) will fail the run." $interleaveArgs += "--fail-on-regression" @@ -726,7 +814,11 @@ if ((-not [string]::IsNullOrEmpty($BaselineLabel)) -and ($RunMode -eq "interleav } elseif (-not [string]::IsNullOrEmpty($BaselineLabel)) { # --- Legacy sequential path: full baseline pass, then full candidate pass, then compare ------- + # -SwitchUnderTest needs a different RUNNER_CONFIG per pass; every other baseline flavour + # keeps using the single ambient RUNNER_CONFIG set above (unchanged behaviour). + if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { $env:RUNNER_CONFIG = $BaselineRunnerConfig } Invoke-PerfPass "baseline" $BaselineProject $BaselineBuildArgs + if (-not [string]::IsNullOrEmpty($SwitchUnderTest)) { $env:RUNNER_CONFIG = $CurrentRunnerConfig } Invoke-PerfPass "current" $PerfProject @() Write-Host "Comparing current branch against baseline $BaselineLabel ..." diff --git a/eng/pipelines/perf/scripts/run-perf-tests.sh b/eng/pipelines/perf/scripts/run-perf-tests.sh index 8d05340342..c0b485dc9b 100755 --- a/eng/pipelines/perf/scripts/run-perf-tests.sh +++ b/eng/pipelines/perf/scripts/run-perf-tests.sh @@ -65,10 +65,22 @@ confirmationRuns="3" useManagedSniOnWindows="" useOptimizedAsyncBehaviour="" useConnectionPoolV2="" +# Alternative to --baseline-version/--baseline-source-ref: an A/B experiment on ONE runner-config +# switch. Both passes build the SAME source; only the named switch differs (baseline=false, +# current=true), which is the only way to compare a switch whose value is latched process-wide (e.g. +# UseConnectionPoolV2 is read and cached the first time a pool is created). Mutually exclusive with +# the other two baseline selectors, and overrides the matching --use-* flag (which would otherwise be +# ambiguous: one value cannot describe two passes). +switchUnderTest="" +# Runner-config switches this script is allowed to A/B. Restricted to a known list so a typo fails +# fast here instead of silently writing an inert key into the runner config and reporting a +# meaningless zero-delta comparison. +SUPPORTED_SWITCHES=("UseConnectionPoolV2" "UseOptimizedAsyncBehaviour" "UseManagedSniOnWindows") usage() { echo "Usage: $0 [--configuration ] [--framework ] [--results-subdir ]" \ - "[--baseline-version | --baseline-source-ref [--baseline-repo-url ]]" \ + "[--baseline-version | --baseline-source-ref [--baseline-repo-url ] |" \ + "--switch-under-test <${SUPPORTED_SWITCHES[*]}>]" \ "[--regression-threshold ] [--fail-on-regression]" \ "[--run-mode interleaved|sequential] [--confirmation-runs ]" \ "[--use-managed-sni-on-windows true|false] [--use-optimized-async-behaviour true|false]" \ @@ -83,6 +95,7 @@ while [[ $# -gt 0 ]]; do --baseline-version) baselineVersion="$2"; shift 2 ;; --baseline-source-ref) baselineSourceRef="$2"; shift 2 ;; --baseline-repo-url) baselineRepoUrl="$2"; shift 2 ;; + --switch-under-test) switchUnderTest="$2"; shift 2 ;; --regression-threshold) regressionThreshold="$2"; shift 2 ;; --fail-on-regression) failOnRegression="true"; shift 1 ;; --run-mode) runMode="$2"; shift 2 ;; @@ -103,12 +116,33 @@ case "${runMode}" in *) echo "ERROR: --run-mode must be 'interleaved' or 'sequential' (got '${runMode}')." >&2 usage; exit 2 ;; esac -# The two baseline selectors describe different builds of the same "baseline" pass, so requesting -# both is always a mistake; fail fast rather than silently honouring one of them. +# The three baseline selectors describe different builds/configs of the same "baseline" pass, so +# requesting more than one is always a mistake; fail fast rather than silently honouring one of them. if [[ -n "${baselineVersion}" && -n "${baselineSourceRef}" ]]; then echo "ERROR: --baseline-version and --baseline-source-ref are mutually exclusive." >&2 usage; exit 2 fi +if [[ -n "${switchUnderTest}" && ( -n "${baselineVersion}" || -n "${baselineSourceRef}" ) ]]; then + echo "ERROR: --switch-under-test is mutually exclusive with --baseline-version and --baseline-source-ref: it compares the SAME source build with one switch flipped, so mixing in a source change would make the delta unattributable." >&2 + usage; exit 2 +fi +if [[ -n "${switchUnderTest}" ]]; then + switchSupported="false" + for supported in "${SUPPORTED_SWITCHES[@]}"; do + [[ "${switchUnderTest}" == "${supported}" ]] && switchSupported="true" + done + if [[ "${switchSupported}" != "true" ]]; then + echo "ERROR: --switch-under-test must be one of: ${SUPPORTED_SWITCHES[*]} (got '${switchUnderTest}')." >&2 + usage; exit 2 + fi + # UseManagedSniOnWindows only selects an implementation on Windows. This is the bash entry + # point, so both passes here would run identical managed SNI and the experiment would spend + # hours to report a ~0% delta that looks like "the switch is free" rather than "not applicable". + if [[ "${switchUnderTest}" == "UseManagedSniOnWindows" ]]; then + echo "ERROR: --switch-under-test UseManagedSniOnWindows is Windows-only; on Linux managed SNI is always used, so baseline and current would be identical. Re-run this experiment on the Windows platform." >&2 + exit 2 + fi +fi if ! [[ "${confirmationRuns}" =~ ^[0-9]+$ ]] || [[ "${confirmationRuns}" -lt 1 ]]; then echo "ERROR: --confirmation-runs must be a positive integer (got '${confirmationRuns}')." >&2 usage; exit 2 @@ -123,6 +157,18 @@ validate_bool() { # $1 = flag name (for the message), $2 = value validate_bool use-managed-sni-on-windows "${useManagedSniOnWindows}" validate_bool use-optimized-async-behaviour "${useOptimizedAsyncBehaviour}" validate_bool use-connection-pool-v2 "${useConnectionPoolV2}" +# --switch-under-test forces its switch explicitly for each pass (baseline=false, current=true), so a +# separately-supplied --use-* flag for that SAME switch would be silently overridden; warn rather +# than let that go unnoticed. Other --use-* flags still apply normally to both passes. +case "${switchUnderTest}" in + UseConnectionPoolV2) conflictingValue="${useConnectionPoolV2}"; conflictingFlag="--use-connection-pool-v2" ;; + UseOptimizedAsyncBehaviour) conflictingValue="${useOptimizedAsyncBehaviour}"; conflictingFlag="--use-optimized-async-behaviour" ;; + UseManagedSniOnWindows) conflictingValue="${useManagedSniOnWindows}"; conflictingFlag="--use-managed-sni-on-windows" ;; + *) conflictingValue=""; conflictingFlag="" ;; +esac +if [[ -n "${conflictingValue}" ]]; then + echo "WARNING: ${conflictingFlag} is ignored when --switch-under-test is ${switchUnderTest} (baseline forces false, current forces true)." >&2 +fi #################################################################################################### # Resolve paths @@ -148,6 +194,7 @@ echo " Framework : ${framework}" echo " Results dir : ${RESULTS_DIR}" echo " Baseline ver : ${baselineVersion:-}" echo " Baseline ref : ${baselineSourceRef:-}" +echo " Switch A/B : ${switchUnderTest:-}${switchUnderTest:+ (baseline=false vs current=true)}" echo " Run mode : ${runMode} (confirmation runs: ${confirmationRuns})" echo " SQL_SERVER : ${SQL_SERVER:-}" echo " PERF_CLIENT_CPUS: ${PERF_CLIENT_CPUS:-}" @@ -295,7 +342,8 @@ export MALLOC_MMAP_THRESHOLD_="${MALLOC_MMAP_THRESHOLD_:-134217728}" # 128 MiB export MALLOC_TRIM_THRESHOLD_="${MALLOC_TRIM_THRESHOLD_:--1}" # never trim # --- §2.9 Network tuning (best-effort; needs privilege, so it must never fail the run) ------------ -# Connection-churn benches (ConnectionPoolStress, ParallelAsyncConnection) exhaust ephemeral ports; +# Connection-churn benches (ConnectionPoolStress, ConnectionPoolRamp, +# ConnectionPoolThreadPoolPressure, ParallelAsyncConnection) exhaust ephemeral ports; # widen the range and allow TIME_WAIT reuse so socket setup latency stays stable. 'sudo -n' keeps # this non-interactive: on a VM without passwordless sudo it fails immediately instead of blocking # on a password prompt, then we fall back to a non-sudo sysctl (and finally give up quietly). @@ -358,10 +406,20 @@ export PERF_CFG_USE_MANAGED_SNI="${useManagedSniOnWindows}" export PERF_CFG_USE_OPTIMIZED_ASYNC="${useOptimizedAsyncBehaviour}" export PERF_CFG_USE_CONNECTION_POOL_V2="${useConnectionPoolV2}" -python3 - "$PERF_DIR/runnerconfig.jsonc" "$RUNNER_CONFIG" <<'PY' +# write_runner_config [switch_name] [switch_value] +# Writes one runner config (checked-in runnerconfig.jsonc + injected connection string + behaviour +# overrides) to . When is given, that config key is forced to +# ("true"/"false") regardless of the corresponding PERF_CFG_* value -- used by --switch-under-test, +# which needs a different value for the same switch in each pass. With no switch name the config is +# built purely from the PERF_CFG_* values, exactly as before. +write_runner_config() { + local dst="$1" + local switch_name="${2:-}" + local switch_value="${3:-}" + python3 - "$PERF_DIR/runnerconfig.jsonc" "$dst" "$switch_name" "$switch_value" <<'PY' import json, os, re, sys -src, dst = sys.argv[1], sys.argv[2] +src, dst, switch_name, switch_value = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] with open(src, "r", encoding="utf-8-sig") as fh: text = fh.read() @@ -387,13 +445,16 @@ cfg["ConnectionString"] = ( # Apply the optional SqlClient behaviour overrides supplied by the pipeline. An empty value leaves # the checked-in default untouched; otherwise the flag is forced to the requested boolean so the -# benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. +# benchmarks run with (and PerfRun.Config records) exactly the requested behaviour. The +# switch-under-test override (when this config names one) takes precedence over the corresponding +# PERF_CFG_* value, so a single checked-in template can be stamped out per pass with that one switch +# flipped and everything else identical. for env_name, cfg_key in ( ("PERF_CFG_USE_MANAGED_SNI", "UseManagedSniOnWindows"), ("PERF_CFG_USE_OPTIMIZED_ASYNC", "UseOptimizedAsyncBehaviour"), ("PERF_CFG_USE_CONNECTION_POOL_V2", "UseConnectionPoolV2"), ): - val = os.environ.get(env_name, "") + val = switch_value if cfg_key == switch_name else os.environ.get(env_name, "") if val != "": cfg[cfg_key] = (val.lower() == "true") @@ -402,6 +463,21 @@ with open(dst, "w", encoding="utf-8") as fh: print(f"Wrote runner config to {dst} (Server=tcp:{server},1433; Initial Catalog={db})") PY +} + +write_runner_config "${RUNNER_CONFIG}" + +# --switch-under-test needs two DIFFERENT runner configs (baseline runs the switch off, current runs +# it on), so stamp out two more copies here alongside the shared one above. Everything else in them +# is identical, so any measured delta is attributable to the switch alone. +BASELINE_RUNNER_CONFIG="" +CURRENT_RUNNER_CONFIG="" +if [[ -n "${switchUnderTest}" ]]; then + BASELINE_RUNNER_CONFIG="${REPO_ROOT}/perf-runnerconfig-baseline.json" + CURRENT_RUNNER_CONFIG="${REPO_ROOT}/perf-runnerconfig-current.json" + write_runner_config "${BASELINE_RUNNER_CONFIG}" "${switchUnderTest}" "false" + write_runner_config "${CURRENT_RUNNER_CONFIG}" "${switchUnderTest}" "true" +fi #################################################################################################### # 4 & 5. Run the benchmarks, pinned to the reserved client CPU set. @@ -686,6 +762,11 @@ elif [[ -n "${baselineSourceRef}" ]]; then prepare_baseline_source "${baselineSourceRef}" baselineLabel="${BASELINE_SRC_LABEL}" baselineProject="${BASELINE_PERF_PROJECT}" +elif [[ -n "${switchUnderTest}" ]]; then + # Switch A/B: SAME source/project for both passes (baselineProject/baselineBuildArgs are already + # the candidate's, set above), so only the runner config differs (see BASELINE_RUNNER_CONFIG / + # CURRENT_RUNNER_CONFIG written above: the named switch off vs on). + baselineLabel="${switchUnderTest}=false" fi # Record the resolved baseline label (for a source baseline this is '@') in the results @@ -702,12 +783,24 @@ if [[ -n "${baselineLabel}" && "${runMode}" == "interleaved" ]]; then # orchestrator run one unit at a time (baseline then candidate) and confirm any flagged # regression across N passes before it counts toward the gate. ################################################################################################ - build_variant "baseline" "${baselineProject}" ${baselineBuildArgs[@]+"${baselineBuildArgs[@]}"} - build_variant "current" "${PERF_PROJECT}" + if [[ -n "${switchUnderTest}" ]]; then + # Switch A/B measures one build against itself with a switch flipped, so building the same + # project twice would just burn several minutes producing identical bits. Build once and + # point both variants at it; the orchestrator runs each variant in its own working directory + # (rep//), so a shared exe dir cannot cross-contaminate their artifacts. + build_variant "current" "${PERF_PROJECT}" + baselineExeDir="${REPO_ROOT}/perf-build-current" + currentExeDir="${REPO_ROOT}/perf-build-current" + else + build_variant "baseline" "${baselineProject}" ${baselineBuildArgs[@]+"${baselineBuildArgs[@]}"} + build_variant "current" "${PERF_PROJECT}" + baselineExeDir="${REPO_ROOT}/perf-build-baseline" + currentExeDir="${REPO_ROOT}/perf-build-current" + fi interleave_args=( - --baseline-exe-dir "${REPO_ROOT}/perf-build-baseline" - --current-exe-dir "${REPO_ROOT}/perf-build-current" + --baseline-exe-dir "${baselineExeDir}" + --current-exe-dir "${currentExeDir}" --assembly "PerformanceTests.dll" --results-dir "${RESULTS_DIR}" --threshold "${regressionThreshold}" @@ -715,6 +808,15 @@ if [[ -n "${baselineLabel}" && "${runMode}" == "interleaved" ]]; then --baseline-version "${baselineLabel}" --client-cpus "${PERF_CLIENT_CPUS:-}" ) + # --switch-under-test: baseline and current subprocesses need DIFFERENT RUNNER_CONFIG values (the + # switch off vs on), even though both are otherwise the same build/env; every other baseline + # flavour keeps sharing the single ambient RUNNER_CONFIG set above. + if [[ -n "${switchUnderTest}" ]]; then + interleave_args+=( + --baseline-runner-config "${BASELINE_RUNNER_CONFIG}" + --current-runner-config "${CURRENT_RUNNER_CONFIG}" + ) + fi if [[ "${failOnRegression}" == "true" ]]; then echo "Regression gate ENABLED: a CONFIRMED candidate-slower regression (> ${regressionThreshold}%) will fail the run." interleave_args+=(--fail-on-regression) @@ -724,7 +826,15 @@ if [[ -n "${baselineLabel}" && "${runMode}" == "interleaved" ]]; then elif [[ -n "${baselineLabel}" ]]; then # --- Legacy sequential path: full baseline pass, then full candidate pass, then compare ------- + # --switch-under-test needs a different RUNNER_CONFIG per pass; every other baseline flavour + # keeps using the single ambient RUNNER_CONFIG exported above (unchanged behaviour). + if [[ -n "${switchUnderTest}" ]]; then + export RUNNER_CONFIG="${BASELINE_RUNNER_CONFIG}" + fi run_pass "baseline" "${baselineProject}" ${baselineBuildArgs[@]+"${baselineBuildArgs[@]}"} + if [[ -n "${switchUnderTest}" ]]; then + export RUNNER_CONFIG="${CURRENT_RUNNER_CONFIG}" + fi run_pass "current" "${PERF_PROJECT}" echo "Comparing current branch against baseline ${baselineLabel} ..." diff --git a/eng/pipelines/perf/sqlclient-perf-experiment.yml b/eng/pipelines/perf/sqlclient-perf-experiment.yml new file mode 100644 index 0000000000..80cf11320c --- /dev/null +++ b/eng/pipelines/perf/sqlclient-perf-experiment.yml @@ -0,0 +1,239 @@ +#################################################################################################### +# Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this +# file to you under the MIT license. See the LICENSE file in the project root for more information. +#################################################################################################### + +# SqlClient Switch Experiment Performance pipeline. +# +# Third sibling of sqlclient-perf-pipeline.yml and sqlclient-perf-pr-pipeline.yml. It runs the SAME +# benchmarks, on the SAME Perf Test Lab extends template (v1/Perf.Test.Job.yml@PerfTemplates), +# through the SAME on-VM scripts. What differs is the QUESTION it answers: +# +# * perf pipeline - "has this branch regressed against a released package?" (source varies) +# * perf-pr pipeline - "has this PR regressed the branch it merges into?" (source varies) +# * THIS pipeline - "what does this AppContext/runner switch cost or buy?" (CONFIG varies) +# +# Both passes here build the SAME source (whatever branch the run is queued on); only ONE runner +# config switch differs between them - baseline runs it OFF, current runs it ON. Two separate +# processes are required because these switches are latched process-wide (e.g. UseConnectionPoolV2 is +# read and cached the first time a connection pool is created), so they cannot be toggled between +# benchmarks inside a single run. Queue this pipeline on a PR branch to ask "what does the switch do +# to my change?", or on main to ask "what does the switch do to main?". +# +# * No Kusto - switch-experiment results are NEVER ingested into the perf database, and this is why +# the mode lives in its own pipeline rather than as a flag on the other two: +# - both PerfRun rows would share one DerivedRunId (driver|commit|pipelineRunId), +# because both passes are the same commit in the same pipeline run; +# - PerfRun.Config is stamped once per run, so the two rows could not record the +# differing switch values that are the entire point of the experiment; +# - nothing marks a row as an experiment, so the switch-ON pass would be +# indistinguishable from an ordinary measurement of the branch and would distort +# the very trends the other two pipelines exist to protect. +# Having no ADX variable group and no translate/ingest steps in this file makes that +# exclusion structural rather than a conditional someone can flip by accident. +# The comparison report and the raw BenchmarkDotNet artifacts are still published. +# +# Everything else - platform/VM provisioning, benchmark run model, noise controls - is identical to +# the other two pipelines; see eng/pipelines/perf/README.md. + +# Set the pipeline run name to the day-of-year and the daily run counter. +name: $(DayOfYear)$(Rev:rr) + +# Manual/queue-time only. A full perf run occupies a dedicated host for hours, and a switch +# experiment is an investigation someone opts into deliberately, never an automatic gate. +pr: none +trigger: none + +parameters: + + # The single runner-config switch under test. Baseline runs it false, current runs it true, so the + # reported delta is "what turning this switch ON does". Restricted to the switches the run scripts + # know how to stamp into the runner config; the scripts re-validate and fail fast on anything else. + # + # UseManagedSniOnWindows additionally requires platform=windows. ADO cannot cross-validate two + # queue-time parameters here (this pipeline extends a template, so there is no step to check it + # before the job starts), so the run scripts reject that combination as their first action. + - name: switchUnderTest + displayName: Switch under test (baseline=off vs current=on) + type: string + default: UseConnectionPoolV2 + values: + - UseConnectionPoolV2 + - UseOptimizedAsyncBehaviour + - UseManagedSniOnWindows + + # Target OS for the perf VM and the benchmark client. + - name: platform + displayName: Platform + type: string + default: linux + values: + - linux + - windows + + # The .NET runtime the benchmarks are executed against. Must be one of the target frameworks of + # the PerformanceTests project (net8.0/net9.0/net10.0). + - name: dotnetFramework + displayName: .NET Framework (TFM) + type: string + default: net9.0 + values: + - net8.0 + - net9.0 + - net10.0 + + # Maximum time (minutes) the template waits for the benchmark run on the VM before timing out. + # Matches the nightly pipeline rather than the PR pipeline: both passes here share ONE build (the + # source is identical on both sides), so there is no second driver build to pay for. + - name: testTimeoutMinutes + displayName: Test Timeout (minutes) + type: number + default: 180 + + # Percent difference (switch-on vs switch-off mean) the comparison flags. + - name: regressionThreshold + displayName: Difference Threshold (%) + type: number + default: 10 + + # Whether a confirmed "switch ON is slower than switch OFF" result should FAIL the run. + # + # NOTE: this maps onto the run scripts' --fail-on-regression gate, but it does NOT mean the same + # thing as it does in the other two pipelines. There, a regression means the branch got worse and + # failing is a quality gate. Here it is a RESULT: the switch made things slower. That is often + # exactly what you queued the run to find out, so this defaults to false and should only be enabled + # when you are asserting "this switch must not be a slowdown" (e.g. before flipping its default). + - name: failIfSwitchSlower + displayName: Fail run if the switch is slower + type: boolean + default: false + + # Benchmark run model (wiki 339 §2.2/§2.3/§2.6): + # interleaved -> run one benchmark unit at a time, switch-off and switch-on back-to-back, and + # confirm any flagged difference across N passes (best-of-N). Noise-resistant + # default, and especially valuable here because the two passes are otherwise + # identical, so any systematic drift between them is pure measurement noise. + # sequential -> legacy: run the whole switch-off suite, then the whole switch-on suite, compare. + - name: benchmarkRunMode + displayName: Benchmark run mode + type: string + default: interleaved + values: + - interleaved + - sequential + + # Best-of-N: total interleaved passes for a flagged unit before a difference is confirmed + # (1 disables confirmation). Only used when benchmarkRunMode = interleaved. + - name: confirmationRuns + displayName: Confirmation runs (best-of-N) + type: number + default: 3 + +# Fixed (non-configurable) constants for this pipeline. These are intentionally NOT parameters or +# library variables: they are invariant for the SqlClient perf pipelines. +# * buildConfiguration = Release - perf numbers are only meaningful in Release. +# * sourcesSubDir = dotnet-sqlclient - folder 'self' checks out into under the template's +# MULTI-REPO checkout ($(Build.SourcesDirectory)/); +# must match the ADO repository name. +# +# The other pipelines additionally expose UseManagedSniOnWindows / UseOptimizedAsyncBehaviour / +# UseConnectionPoolV2 as queue-time flags applied to BOTH passes. This pipeline deliberately does +# not: an experiment that varies one switch while others are also moved off their checked-in defaults +# produces a delta nobody can attribute. Every switch except the one under test is therefore left at +# its runnerconfig.jsonc default (which is what those parameters' defaults already are). +# +# NOTE: like sqlclient-perf-pr-pipeline.yml, this pipeline does NOT reference the 'ADX Cluster +# Variables' group and has no translate/ingest steps. See the header comment for why that is +# structural here rather than conditional. +variables: + + # Pre-computed testScriptArgs chunks. The Windows entry point is PowerShell and binds + # '-PascalCase' params, while the Linux one is bash and parses '--kebab-case' flags, so the + # argument STYLE differs by platform. Assembling the final args from these per-platform chunks + # keeps the single testScriptArgs below readable. No baseline-selector args appear here at all: + # --switch-under-test IS the baseline selector for this pipeline, and the run scripts reject it + # being combined with --baseline-version / --baseline-source-ref. + - ${{ if eq(parameters.platform, 'windows') }}: + - name: PerfArgsCommon + value: '-Configuration Release -Framework ${{ parameters.dotnetFramework }} -ResultsSubdir perf-results -RegressionThreshold ${{ parameters.regressionThreshold }} -RunMode ${{ parameters.benchmarkRunMode }} -ConfirmationRuns ${{ parameters.confirmationRuns }} -SwitchUnderTest ${{ parameters.switchUnderTest }}' + - ${{ else }}: + - name: PerfArgsCommon + value: '--configuration Release --framework ${{ parameters.dotnetFramework }} --results-subdir perf-results --regression-threshold ${{ parameters.regressionThreshold }} --run-mode ${{ parameters.benchmarkRunMode }} --confirmation-runs ${{ parameters.confirmationRuns }} --switch-under-test ${{ parameters.switchUnderTest }}' + + # Gate flag, only when the run is asserting the switch must not be a slowdown. + - ${{ if and(eq(parameters.platform, 'windows'), eq(parameters.failIfSwitchSlower, true)) }}: + - name: PerfArgsFail + value: '-FailOnRegression' + - ${{ elseif eq(parameters.failIfSwitchSlower, true) }}: + - name: PerfArgsFail + value: '--fail-on-regression' + - ${{ else }}: + - name: PerfArgsFail + value: '' + +# Reference the PerfTest repository that hosts the reusable extends template. Driver-team projects +# have been onboarded with the Agent Pools and Service Connections required to consume it. +resources: + repositories: + - repository: PerfTemplates + type: git + name: InternalDriverTools/PerfTest + ref: refs/heads/main + +# Consume the Perf Test Lab template. The template defines the whole job/stage structure; we only +# pass parameters into it. +extends: + template: v1/Perf.Test.Job.yml@PerfTemplates + parameters: + platform: ${{ parameters.platform }} + + # The entire driver source tree is copied to the VM so the benchmarks (which reference + # Microsoft.Data.SqlClient as a project) can be built from source against the current commit. + # Under the template's multi-repo checkout, 'self' lands in a repo-named subfolder. + testRootDir: $(Build.SourcesDirectory)/dotnet-sqlclient + + # Entry-point script (relative to testRootDir), selected by platform. Linux uses bash, Windows + # uses PowerShell, per the template's .sh/.ps1 convention. Same scripts as the other two perf + # pipelines: the switch experiment is just a different baseline selector. + ${{ if eq(parameters.platform, 'windows') }}: + testScript: eng/pipelines/perf/scripts/run-perf-tests.ps1 + ${{ else }}: + testScript: eng/pipelines/perf/scripts/run-perf-tests.sh + + # Arguments forwarded to the script. The script also reads SQL_SERVER, SQL_PASSWORD and + # PERF_CLIENT_CPUS directly from the VM session environment (injected by the template). Empty + # chunks collapse to harmless extra spaces that both PowerShell and bash arg parsing ignore. + testScriptArgs: '$(PerfArgsCommon) $(PerfArgsFail)' + + # Subfolder (relative to testRootDir on the VM) the script writes results into. The template + # copies this VM folder back to the agent, but always lands it at a fixed location: + # $(Build.ArtifactStagingDirectory)/results (and publishes it as the 'perf-results' artifact). + testResultsSubDir: perf-results + + testTimeoutMinutes: ${{ parameters.testTimeoutMinutes }} + jobName: SqlClientPerfExperiment + + # Post-test steps run on the agent after results have been copied back. The template already + # publishes the results artifact and attaches any top-level results/*.md as run summaries; this + # step additionally surfaces the BenchmarkDotNet markdown reports in the build log. There is no + # Kusto translation/ingestion here by design - see the header comment. + steps: + # Tag the build with the switch that was measured, so switch experiments are identifiable at a + # glance in the ADO build list and are never mistaken for an ordinary baseline comparison. + # Unlike the PR pipeline there is nothing to read back from the VM: the switch name is fixed at + # queue time, and both passes are the commit the run was queued on. + # NOTE: the tag intentionally has no ':' - build tags are placed in the request URL path and a + # colon trips ADO's "potentially dangerous Request.Path" filter. + - bash: | + echo "##vso[build.addbuildtag]Switch ${{ parameters.switchUnderTest }}" + displayName: 'Tag build with switch under test' + condition: succeededOrFailed() + + - task: Bash@3 + displayName: 'Show performance results' + condition: succeededOrFailed() + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)/dotnet-sqlclient/eng/pipelines/perf/scripts/show_perf_results.sh + env: + RESULTS_DIR: $(Build.ArtifactStagingDirectory)/results diff --git a/eng/pipelines/pr/jobs/test-buildproj-job.yml b/eng/pipelines/pr/jobs/test-buildproj-job.yml index c232513e1f..1e9a93fdab 100644 --- a/eng/pipelines/pr/jobs/test-buildproj-job.yml +++ b/eng/pipelines/pr/jobs/test-buildproj-job.yml @@ -45,7 +45,7 @@ parameters: type: string # Name of the image in the customized ADO pool to use to execute this job. - - name: platformImage + - name: poolImage type: string # Name of the pool to use for jobs that require customized VM images. @@ -89,7 +89,7 @@ jobs: pool: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.platformImage }} + - imageOverride -equals ${{ parameters.poolImage }} steps: # Install dotnet and the runtime that will run the tests (if it is not netframework) diff --git a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml index 38b49695fe..3b975aa46b 100644 --- a/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml +++ b/eng/pipelines/pr/jobs/test-sqlclientmanual-job.yml @@ -49,7 +49,7 @@ parameters: type: string # Name of the image in the customized ADO pool to use to execute this job. - - name: platformImage + - name: poolImage type: string # General name of the operating system that will be used to execute the job. @@ -132,7 +132,7 @@ jobs: pool: name: ${{ parameters.poolName }} demands: - - imageOverride -equals ${{ parameters.platformImage }} + - imageOverride -equals ${{ parameters.poolImage }} variables: # Bring the generated SA password from the secrets stage into scope for this test stage. diff --git a/eng/pipelines/pr/sqlclient-pr-pipeline.yml b/eng/pipelines/pr/sqlclient-pr-pipeline.yml index b72e15ef53..eab08243e8 100644 --- a/eng/pipelines/pr/sqlclient-pr-pipeline.yml +++ b/eng/pipelines/pr/sqlclient-pr-pipeline.yml @@ -70,32 +70,32 @@ parameters: default: - displayName: "windows_net462" dotnet: "net462" - image: "ADO-MMS22-SQL22" + image: "ADO-MMS25-SQL25" operatingSystem: "Windows" - displayName: "windows_net8" dotnet: "net8.0" - image: "ADO-MMS22-SQL22" + image: "ADO-MMS25-SQL25" operatingSystem: "Windows" - displayName: "windows_net9" dotnet: "net9.0" - image: "ADO-MMS22-SQL22" + image: "ADO-MMS25-SQL25" operatingSystem: "Windows" - displayName: "windows_net10" dotnet: "net10.0" - image: "ADO-MMS22-SQL22" + image: "ADO-MMS25-SQL25" operatingSystem: "Windows" - displayName: "linux_net8" dotnet: "net8.0" - image: "ADO-UB22-SQL22" + image: "ADO-UB24-SQL25" operatingSystem: "Linux" - displayName: "linux_net9" dotnet: "net9.0" - image: "ADO-UB22-SQL22" + image: "ADO-UB24-SQL25" operatingSystem: "Linux" - displayName: "linux_net10" dotnet: "net10.0" - image: "ADO-UB22-SQL22" + image: "ADO-UB24-SQL25" operatingSystem: "Linux" variables: @@ -103,6 +103,16 @@ variables: - template: /eng/pipelines/pr/variables/pr-variables.yml@self stages: + # NOTE: '$(general_purpose_pool_name)' comes from the 'sqlclient-pipeline-config-v1' + # variable group, which is imported by + # /eng/pipelines/pr/variables/pr-variables.yml (included above). It is + # referenced here, at the pipeline root, and passed down to every stage as a + # parameter, so that no template reads the pool name from a variable group + # directly. + # + # The same variable group and variable are used by the CI pipelines; see + # /eng/pipelines/dotnet-sqlclient-ci-core.yml. + # Stage 1a: Build and pack all projects in the repository - template: /eng/pipelines/pr/stages/pack-stage.yml@self parameters: @@ -110,11 +120,15 @@ stages: buildSuffix: pr stageName: ${{ variables.stageNamePack }} packArtifactBaseName: ${{ variables.packArtifactBaseName }} + poolName: $(general_purpose_pool_name) + poolImage: ADO-UB24 # Stage 1b: Generate secrets - template: /eng/pipelines/pr/stages/generate-secrets-stage.yml@self parameters: stageName: ${{ variables.stageNameSecrets }} + poolName: $(general_purpose_pool_name) + poolImage: ADO-UB24 # Stage 2: Execute tests and collect code coverage - template: /eng/pipelines/pr/stages/test-stages.yml@self @@ -122,7 +136,7 @@ stages: buildConfiguration: Debug buildSuffix: pr dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - poolName: $(PoolNameDefault) + poolName: $(general_purpose_pool_name) stageNamePack: ${{ variables.stageNamePack }} stageNameSecrets: ${{ variables.stageNameSecrets }} testResultsArtifactBaseName: ${{ variables.testResultsArtifactBaseName }} @@ -144,6 +158,8 @@ stages: - template: /eng/pipelines/pr/stages/collect-coverage-stage.yml@self parameters: coverageArtifactBaseName: ${{ variables.coverageArtifactBaseName }} + poolName: $(general_purpose_pool_name) + poolImage: ADO-UB24 dependsOn: - ${{ each platform in parameters.platforms }}: - "test_${{ platform.displayName }}" diff --git a/eng/pipelines/pr/stages/collect-coverage-stage.yml b/eng/pipelines/pr/stages/collect-coverage-stage.yml index 9a0328ec50..9fa4c1828a 100644 --- a/eng/pipelines/pr/stages/collect-coverage-stage.yml +++ b/eng/pipelines/pr/stages/collect-coverage-stage.yml @@ -16,6 +16,14 @@ parameters: - name: coverageArtifactBaseName type: string + # The name of the 1ES pool to run in. + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + stages: - stage: collect_code_coverage displayName: "Collect code coverage" @@ -26,7 +34,10 @@ stages: displayName: "Collect Code Coverage" pool: - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: diff --git a/eng/pipelines/pr/stages/generate-secrets-stage.yml b/eng/pipelines/pr/stages/generate-secrets-stage.yml index 20a81fe626..f936594e1a 100644 --- a/eng/pipelines/pr/stages/generate-secrets-stage.yml +++ b/eng/pipelines/pr/stages/generate-secrets-stage.yml @@ -30,6 +30,14 @@ parameters: - name: stageName type: string + # The name of the 1ES pool to run in. + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + stages: # The stage downstream stages must depend on to ensure the secrets are generated before they are @@ -45,9 +53,11 @@ stages: - job: secrets_job displayName: Generate Secrets pool: - # We don't need anything special, so use the standard Microsoft-hosted Ubuntu image, which - # is typically very fast to spin up. - vmImage: ubuntu-latest + # This job has no special image requirements, so use a minimal Linux image. + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/pr/stages/pack-stage.yml b/eng/pipelines/pr/stages/pack-stage.yml index 2b43c8632a..a142eac5e9 100644 --- a/eng/pipelines/pr/stages/pack-stage.yml +++ b/eng/pipelines/pr/stages/pack-stage.yml @@ -42,6 +42,14 @@ parameters: - name: packArtifactBaseName type: string + # The name of the 1ES pool to run in. + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + stages: - stage: ${{ parameters.stageName }} displayName: Build and Pack Projects @@ -52,7 +60,10 @@ stages: displayName: Build and Pack Projects pool: - vmImage: 'ubuntu-latest' + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} steps: # Install dotnet @@ -136,7 +147,3 @@ stages: artifact: ${{ parameters.packArtifactBaseName }}_attempt$(System.JobAttempt) displayName: Publish Build Output condition: succeededOrFailed() - - - - diff --git a/eng/pipelines/pr/stages/test-stages.yml b/eng/pipelines/pr/stages/test-stages.yml index 0ad9921ec6..ce4c5c32f8 100644 --- a/eng/pipelines/pr/stages/test-stages.yml +++ b/eng/pipelines/pr/stages/test-stages.yml @@ -133,7 +133,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} @@ -148,7 +148,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} @@ -163,7 +163,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} @@ -178,7 +178,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} @@ -197,7 +197,7 @@ stages: platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} platformOperatingSystem: ${{ platform.operatingSystem }} poolName: ${{ parameters.poolName }} @@ -232,7 +232,7 @@ stages: platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} platformOperatingSystem: ${{ platform.operatingSystem }} poolName: ${{ parameters.poolName }} @@ -261,7 +261,7 @@ stages: platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} platformOperatingSystem: ${{ platform.operatingSystem }} poolName: ${{ parameters.poolName }} @@ -290,7 +290,7 @@ stages: platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} platformOperatingSystem: ${{ platform.operatingSystem }} poolName: ${{ parameters.poolName }} @@ -316,7 +316,7 @@ stages: dotnetVerbosity: ${{ parameters.dotnetVerbosity }} platformDisplayName: ${{ platform.displayName }} platformDotnet: ${{ platform.dotnet }} - platformImage: ${{ platform.image }} + poolImage: ${{ platform.image }} poolName: ${{ parameters.poolName }} testResultsArtifactBaseName: ${{ parameters.testResultsArtifactBaseName }} diff --git a/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml b/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml index 8eb8cc40d6..c240b8525f 100644 --- a/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml +++ b/eng/pipelines/pr/steps/configure-sqlserver-linux-step.yml @@ -4,8 +4,8 @@ # See the LICENSE file in the project root for more information. # ################################################################################# -# This step configures an existing SQL Server running on the local Linux host. For example, our 1ES -# Hosted Pool has images like ADO-UB20-SQL22 that come with SQL Server 2022 pre-installed and +# This step configures an existing SQL Server running on the local Linux host. For example, our +# 1ES pools have images like ADO-UB24-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: @@ -16,6 +16,9 @@ parameters: steps: + # Make sure sqlcmd is available; the SQL Server images do not always ship it. + - template: /eng/pipelines/common/templates/steps/install-sqlcmd-linux-step.yml@self + # Configure SQL Server. - bash: | set -u @@ -30,15 +33,8 @@ steps: exit 1 fi - SQLCMD_BIN="" - if command -v sqlcmd >/dev/null 2>&1; then - SQLCMD_BIN="$(command -v sqlcmd)" - elif [ -x /opt/mssql-tools18/bin/sqlcmd ]; then - SQLCMD_BIN="/opt/mssql-tools18/bin/sqlcmd" - elif [ -x /opt/mssql-tools/bin/sqlcmd ]; then - SQLCMD_BIN="/opt/mssql-tools/bin/sqlcmd" - else - echo "ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations." + if [ -z "${SQLCMD_BIN:-}" ]; then + echo "ERROR: sqlcmd was not resolved by the 'Install sqlcmd [Linux]' step." exit 1 fi @@ -79,6 +75,7 @@ steps: -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -Q "SELECT @@VERSION" 2>/dev/null errstatus=$? ((counter++)) @@ -91,6 +88,9 @@ steps: exit $errstatus fi displayName: 'Configure SQL Server [Linux]' + env: + SQLCMD_BIN: $(SqlCmdBin) + SQLCMD_TRUST_ARG: $(SqlCmdTrustArg) - bash: | set -u @@ -98,15 +98,8 @@ steps: SCRIPT_PATH="$(Build.SourcesDirectory)/tools/testsql/createNorthwindDb.sql" MSSQL_SA_PW="${{ parameters.saPassword }}" - SQLCMD_BIN="" - if command -v sqlcmd >/dev/null 2>&1; then - SQLCMD_BIN="$(command -v sqlcmd)" - elif [ -x /opt/mssql-tools18/bin/sqlcmd ]; then - SQLCMD_BIN="/opt/mssql-tools18/bin/sqlcmd" - elif [ -x /opt/mssql-tools/bin/sqlcmd ]; then - SQLCMD_BIN="/opt/mssql-tools/bin/sqlcmd" - else - echo "ERROR: 'sqlcmd' was not found on PATH or in the standard mssql-tools locations." + if [ -z "${SQLCMD_BIN:-}" ]; then + echo "ERROR: sqlcmd was not resolved by the 'Install sqlcmd [Linux]' step." exit 1 fi @@ -114,6 +107,7 @@ steps: -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -Q "IF DB_ID(N'Northwind') IS NOT NULL BEGIN ALTER DATABASE [Northwind] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [Northwind]; END" \ -b @@ -121,7 +115,11 @@ steps: -S localhost \ -U SA \ -P "$MSSQL_SA_PW" \ + ${SQLCMD_TRUST_ARG:-} \ -i "$SCRIPT_PATH" \ -b displayName: 'Create Northwind Database [Linux]' retryCountOnTaskFailure: 1 + env: + SQLCMD_BIN: $(SqlCmdBin) + SQLCMD_TRUST_ARG: $(SqlCmdTrustArg) diff --git a/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml b/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml index 8e07120b4b..cf66c1e9c3 100644 --- a/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml +++ b/eng/pipelines/pr/steps/configure-sqlserver-windows-step.yml @@ -5,7 +5,7 @@ ################################################################################# # This step configures an existing SQL Server running on the local Windows host. For example, our -# 1ES Hosted Pool has images like ADO-MMS22-SQL22 that come with SQL Server 2022 pre-installed and +# 1ES pools have images like ADO-MMS25-SQL25 that come with SQL Server 2025 pre-installed and # running. parameters: diff --git a/eng/pipelines/pr/variables/pr-variables.yml b/eng/pipelines/pr/variables/pr-variables.yml index 5d856c4859..3e97ccb8ac 100644 --- a/eng/pipelines/pr/variables/pr-variables.yml +++ b/eng/pipelines/pr/variables/pr-variables.yml @@ -23,6 +23,9 @@ variables: # UserManagedIdentityClientId - group: sqlclient-testconfig-v1 + # Provides 'general_purpose_pool_name', the 1ES pool that hosts most jobs in this project. + - group: sqlclient-pipeline-config-v1 + # General Variables ====================================================== # Name to use for the packaging stage. This is to ensure that the packaging stage and the test diff --git a/eng/pipelines/scripts/Install-DockerCli.macos.ps1 b/eng/pipelines/scripts/Install-DockerCli.macos.ps1 new file mode 100644 index 0000000000..bdaf588e5c --- /dev/null +++ b/eng/pipelines/scripts/Install-DockerCli.macos.ps1 @@ -0,0 +1,225 @@ +<# +.SYNOPSIS + Installs the docker CLI on an Intel macOS agent from a Homebrew bottle. + +.DESCRIPTION + Homebrew ships no Intel macOS bottle for the current docker formula, so + 'brew install docker' compiles the CLI - and builds Go in order to do it - + which takes minutes and routinely exhausts the pipeline step timeout. + + This installs the newest docker version that *is* bottled for Intel macOS, + read from Homebrew's own OCI registry on ghcr.io. Bottles are + content-addressed, so the download is verified against the digest the + registry advertises rather than a checksum pinned here that someone has to + remember to bump. Nothing is downloaded from outside Homebrew, and the + result is exactly what 'brew install' would have produced. + + Only the docker CLI is handled this way. colima and lima still install + through brew: their bottles carry payloads outside bin/ and colima depends + on lima at runtime, so neither can be installed by lifting a single binary. + +.PARAMETER DestinationPath + Directory the docker binary is written to. Prepended to PATH for subsequent + pipeline steps. + +.NOTES + Intel macOS only. On any other architecture this fails rather than + installing a binary the agent cannot run. +#> +param( + [string]$DestinationPath = (Join-Path -Path $HOME -ChildPath '.docker-cli/bin') +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# tar's exit code is checked explicitly below so the failure carries the member +# path rather than PowerShell's generic native-command error. +$PSNativeCommandUseErrorActionPreference = $false + +$Registry = 'https://ghcr.io' +$RegistryRepository = 'homebrew/core/docker' + +# Newest first. A bottle built for an older macOS still runs on a newer one, so +# any of these work on the current agents. +$PreferredCodenames = @('sequoia', 'sonoma', 'ventura') + +#region Helper Functions + +function Get-RegistryToken { + <# + .SYNOPSIS + An anonymous pull token for the formula's registry repository. + #> + + $uri = "$Registry/token?service=ghcr.io&scope=repository:${RegistryRepository}:pull" + return (Invoke-RestMethod -Uri $uri -MaximumRetryCount 3 -RetryIntervalSec 5).token +} + +function Get-BottleVersion { + <# + .SYNOPSIS + Every docker version tag in the registry, newest first. + #> + param( + [Parameter(Mandatory)][string]$Token + ) + + $uri = "$Registry/v2/$RegistryRepository/tags/list?n=1000" + $tags = @() + + while ($uri) { + $response = Invoke-RestMethod ` + -Uri $uri ` + -Headers @{ Authorization = "Bearer $Token" } ` + -MaximumRetryCount 3 -RetryIntervalSec 5 ` + -ResponseHeadersVariable 'responseHeaders' + + $tags += $response.tags + + # ghcr.io orders tags lexically, which puts the newest on the last page. + $link = if ($responseHeaders.ContainsKey('Link')) { @($responseHeaders['Link'])[0] } else { $null } + $uri = if ($link -match '<([^>]+)>') { $Registry + $Matches[1] } else { $null } + } + + return $tags | + Where-Object { $_ -match '^\d+\.\d+\.\d+(-\d+)?$' } | + Sort-Object -Property { [version]($_ -replace '-', '.') } -Descending +} + +function Get-BottleRefName { + <# + .SYNOPSIS + The bottle ref name an OCI index entry advertises, or $null. + #> + param( + [Parameter(Mandatory)]$IndexEntry + ) + + $annotations = $IndexEntry.PSObject.Properties['annotations'] + if (-not $annotations) { return $null } + + $refName = $annotations.Value.PSObject.Properties['org.opencontainers.image.ref.name'] + if (-not $refName) { return $null } + + return $refName.Value +} + +function Find-Bottle { + <# + .SYNOPSIS + The newest given version carrying an Intel macOS bottle, with that + bottle's manifest digest. + #> + param( + [Parameter(Mandatory)][string]$Token, + [Parameter(Mandatory)][string[]]$Version + ) + + $headers = @{ + Authorization = "Bearer $Token" + Accept = 'application/vnd.oci.image.index.v1+json' + } + + foreach ($candidate in $Version) { + $index = Invoke-RestMethod ` + -Uri "$Registry/v2/$RegistryRepository/manifests/$candidate" ` + -Headers $headers ` + -MaximumRetryCount 3 -RetryIntervalSec 5 + + foreach ($codename in $PreferredCodenames) { + foreach ($entry in $index.manifests) { + # The index already belongs to this version, so only the + # platform component needs matching. Splitting on '.' is what + # keeps 'sonoma' from also matching 'arm64_sonoma', and rejects + # the linux and ':all' refs outright. + $ref = Get-BottleRefName -IndexEntry $entry + if ($ref -and ($ref.Split('.') -contains $codename)) { + return [pscustomobject]@{ + Version = $candidate + Digest = $entry.digest + Ref = $ref + } + } + } + } + } + + throw "No Intel macOS docker bottle in any of the $($Version.Count) published versions." +} + +function Save-BottleBlob { + <# + .SYNOPSIS + Downloads a bottle archive and verifies it against its layer digest. + #> + param( + [Parameter(Mandatory)][string]$Token, + [Parameter(Mandatory)][string]$ManifestDigest, + [Parameter(Mandatory)][string]$Path + ) + + $manifest = Invoke-RestMethod ` + -Uri "$Registry/v2/$RegistryRepository/manifests/$ManifestDigest" ` + -Headers @{ + Authorization = "Bearer $Token" + Accept = 'application/vnd.oci.image.manifest.v1+json' + } ` + -MaximumRetryCount 3 -RetryIntervalSec 5 + + $layerDigest = $manifest.layers[0].digest + + Invoke-WebRequest ` + -Uri "$Registry/v2/$RegistryRepository/blobs/$layerDigest" ` + -Headers @{ Authorization = "Bearer $Token" } ` + -MaximumRetryCount 3 -RetryIntervalSec 5 ` + -OutFile $Path + + $expected = $layerDigest -replace '^sha256:', '' + $actual = (Get-FileHash -Path $Path -Algorithm SHA256).Hash + + # -ne is case-insensitive, so Get-FileHash's uppercase output compares equal. + if ($actual -ne $expected) { + throw "Bottle digest mismatch: expected $expected, got $actual." + } +} + +#endregion Helper Functions + +$architecture = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture +if ($architecture -ne 'X64') { + throw "This installs an Intel macOS bottle, but the agent architecture is $architecture." +} + +$token = Get-RegistryToken +# Every version scanned costs a registry round-trip (~0.2s), but the whole list +# is walked rather than a fixed window: Homebrew has stopped publishing Intel +# bottles, so the newest usable version sinks further down the list over time. +$versions = @(Get-BottleVersion -Token $token) +$bottle = Find-Bottle -Token $token -Version $versions + +$archive = Join-Path ([System.IO.Path]::GetTempPath()) "docker-bottle-$([guid]::NewGuid().ToString('n')).tar.gz" + +try { + Save-BottleBlob -Token $token -ManifestDigest $bottle.Digest -Path $archive + + New-Item -ItemType Directory -Path $DestinationPath -Force | Out-Null + + # Homebrew keeps the cellar directory at the plain version even for a + # revision build, so the '29.7.2-1' bottle unpacks from 'docker/29.7.2'. + $cellarVersion = $bottle.Version -replace '-\d+$', '' + $member = "docker/$cellarVersion/bin/docker" + + # Naming the one member we want is what keeps the rest of the archive - an + # anonymous download - from ever being written to disk. + tar -xz -f $archive -C $DestinationPath --strip-components 3 $member + if ($LASTEXITCODE -ne 0) { + throw "Extracting $member from the docker bottle failed (tar exit $LASTEXITCODE)." + } +} +finally { + Remove-Item -LiteralPath $archive -Force -ErrorAction SilentlyContinue +} + +Write-Host "Installed docker $($bottle.Version) ($($bottle.Ref)) to $DestinationPath" +Write-Host "##vso[task.prependpath]$DestinationPath" diff --git a/eng/pipelines/scripts/Open-LocalizationPr.ps1 b/eng/pipelines/scripts/Open-LocalizationPr.ps1 new file mode 100644 index 0000000000..0e7411ec7e --- /dev/null +++ b/eng/pipelines/scripts/Open-LocalizationPr.ps1 @@ -0,0 +1,454 @@ +<# +.SYNOPSIS + Publishes OneLocBuild-generated localized resource files to GitHub as a + single, continuously-updated pull request. + +.DESCRIPTION + The Localization-CI pipeline regenerates the localized `Strings.*.resx` + files on every scheduled run. Because the generated content is identical + until a previous localization PR is merged, naively opening a new PR per + run produces a pile of byte-for-byte duplicate PRs. + + This script makes the publish step idempotent: + + 1. Clones the target GitHub repository at the base branch. + 2. Copies the freshly generated localized resource files into the clone. + 3. If the content matches the base branch, exits without doing anything. + 4. Otherwise commits onto a *stable* branch name, force-pushes it, and + reuses the existing open pull request if one is already present. + + The net effect is at most one open localization PR at any time. Subsequent + runs refresh that PR in place instead of opening another one. + +.PARAMETER GitHubRepository + The target repository in "owner/repo" form. A trailing ".git" is tolerated + so the existing $(GitHubRepository) pipeline variable can be passed as-is. + +.PARAMETER AccessToken + A GitHub token with "contents: write" and "pull requests: write" on the + target repository. Defaults to the GITHUB_TOKEN environment variable. + +.PARAMETER SourceDirectory + The directory holding the OneLocBuild output. Localized files are resolved + relative to this path using ResourcesPath. Defaults to the current + directory. + +.PARAMETER WorkingDirectory + The directory the target repository is cloned into. Defaults to a new + "loc-pr-" folder under the system temp path, which is removed when + the script finishes. + +.PARAMETER BaseBranch + The branch the pull request targets. Defaults to "main". + +.PARAMETER BranchName + The stable branch the localized files are published to. Reusing one branch + name across runs is what allows the pull request to be reused. Defaults to + "dev/automation/onelocbuild". + +.PARAMETER ResourcesPath + Repository-relative path to the resources folder. Defaults to the + Microsoft.Data.SqlClient resources folder. + +.PARAMETER ResourceFilePattern + Filename pattern for the localized resource files to publish. Defaults to + "Strings.*.resx", which matches the localized files but not the English + "Strings.resx" source. + +.PARAMETER DryRun + Performs every read-only step - clone, copy, change detection and the + existing-pull-request lookup - but does not push the branch and does not + create or update a pull request. Use this to validate pipeline wiring + (paths, token scopes, OneLocBuild output) without publishing anything. + +.NOTES + Intended to be invoked from the internal Localization-CI pipeline. It is + safe to re-run: with no localization changes pending it is a no-op, and + with changes pending it converges on a single open pull request. +#> + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# See the LICENSE file in the project root for more information. + +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [string]$GitHubRepository, + + [string]$AccessToken = $env:GITHUB_TOKEN, + + [string]$SourceDirectory = (Get-Location).Path, + + [string]$WorkingDirectory, + + [string]$BaseBranch = 'main', + + [string]$BranchName = 'dev/automation/onelocbuild', + + [string]$ResourcesPath = 'src/Microsoft.Data.SqlClient/src/Resources', + + [string]$ResourceFilePattern = 'Strings.*.resx', + + [switch]$DryRun +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Several git invocations below use the exit code as a signal (for example +# "git diff --cached --quiet" returns 1 when there are staged changes). On +# PowerShell 7.3+ that would otherwise be turned into a terminating error. +$PSNativeCommandUseErrorActionPreference = $false + +$PrTitle = '[Scheduled Run] Localized resource files from OneLocBuild' + +#region Helper Functions + +function Get-RepositorySlug { + <# + .SYNOPSIS + Normalizes a repository reference into "owner/repo" form. + #> + param( + [Parameter(Mandatory)][AllowEmptyString()][string]$Repository + ) + + $slug = $Repository.Trim().TrimEnd('/') + if ($slug.EndsWith('.git', [StringComparison]::OrdinalIgnoreCase)) { + $slug = $slug.Substring(0, $slug.Length - 4) + } + + if ($slug -notmatch '^[^/\s]+/[^/\s]+$') { + throw "GitHubRepository must be in 'owner/repo' form, but was '$Repository'." + } + + return $slug +} + +function Get-PullRequestBody { + <# + .SYNOPSIS + Builds the pull request description. + #> + param( + [Parameter(Mandatory)][string]$Branch, + [Parameter(Mandatory)][datetime]$UpdatedUtc + ) + + $timestamp = $UpdatedUtc.ToString('yyyy-MM-dd HH:mm') + + return @" +Automated PR created from the OneLocBuild scheduled pipeline run. + +Contains updated localized ``Strings.*.resx`` resource files. + +This pull request is refreshed in place by every scheduled localization run, so +there is only ever one open localization PR. Merging it lets the next run start +from a clean base; leaving it open simply keeps it up to date. + +- Branch: ``$Branch`` +- Last updated: $timestamp UTC +"@ +} + +function Invoke-GitHubApi { + <# + .SYNOPSIS + Calls a GitHub REST API endpoint, surfacing the response body on error. + #> + param( + [Parameter(Mandatory)][string]$Uri, + [string]$Method = 'GET', + [object]$Body = $null, + [Parameter(Mandatory)][hashtable]$Headers + ) + + $params = @{ + Uri = $Uri + Method = $Method + Headers = $Headers + } + + if ($null -ne $Body) { + $params['Body'] = ($Body | ConvertTo-Json -Depth 10 -Compress) + $params['ContentType'] = 'application/json' + } + + try { + return Invoke-RestMethod @params -ErrorAction Stop + } + catch { + $status = $null + $responseBody = '' + + # PowerShell 7 already carries the response body on the error record and + # exposes an HttpResponseMessage, which has no GetResponseStream(). Windows + # PowerShell leaves ErrorDetails empty but does expose the stream, so try + # the error record first and keep the stream as a fallback. + if ($null -ne $_.ErrorDetails -and -not [string]::IsNullOrWhiteSpace($_.ErrorDetails.Message)) { + $responseBody = $_.ErrorDetails.Message + } + + if ($_.Exception.PSObject.Properties.Name -contains 'Response' -and $null -ne $_.Exception.Response) { + $status = $_.Exception.Response.StatusCode.value__ + + if ([string]::IsNullOrWhiteSpace($responseBody)) { + try { + $stream = $_.Exception.Response.GetResponseStream() + if ($null -ne $stream) { + $reader = New-Object System.IO.StreamReader($stream) + try { $responseBody = $reader.ReadToEnd() } finally { $reader.Dispose() } + } + } + catch { + # No readable stream (PowerShell 7) or it was already consumed; + # the status code alone is still useful. + } + } + } + + Write-Host "##vso[task.logissue type=error]GitHub API $Method $Uri failed (HTTP $status)." + if ($responseBody) { + Write-Host "##vso[task.logissue type=error]Response: $responseBody" + } + + throw + } +} + +function Invoke-Git { + <# + .SYNOPSIS + Runs a git command and throws if it reports failure. + #> + param( + [Parameter(Mandatory, ValueFromRemainingArguments)][string[]]$Arguments + ) + + & git @Arguments + if ($LASTEXITCODE -ne 0) { + throw "git $($Arguments -join ' ') failed with exit code $LASTEXITCODE." + } +} + +#endregion + +#region Validation + +if ([string]::IsNullOrWhiteSpace($AccessToken)) { + throw 'A GitHub access token is required. Set GITHUB_TOKEN or pass -AccessToken.' +} + +$repoSlug = Get-RepositorySlug -Repository $GitHubRepository +$repoOwner = $repoSlug.Split('/')[0] + +$sourceResources = Join-Path $SourceDirectory $ResourcesPath +if (-not (Test-Path -LiteralPath $sourceResources)) { + throw "Localized resources folder not found at '$sourceResources'." +} + +$localizedFiles = @(Get-ChildItem -Path $sourceResources -Filter $ResourceFilePattern -File) +if ($localizedFiles.Count -eq 0) { + throw "No files matching '$ResourceFilePattern' were found in '$sourceResources'. Did the OneLocBuild step run?" +} + +$ownedWorkingDirectory = $false +if ([string]::IsNullOrWhiteSpace($WorkingDirectory)) { + $WorkingDirectory = Join-Path ([System.IO.Path]::GetTempPath()) "loc-pr-$([guid]::NewGuid().ToString('n'))" + $ownedWorkingDirectory = $true +} + +$headers = @{ + 'Authorization' = "Bearer $AccessToken" + 'Accept' = 'application/vnd.github+json' + 'X-GitHub-Api-Version' = '2022-11-28' + 'User-Agent' = 'SqlClient-DevOps' +} + +#endregion + +Write-Host '=== Publish localized resources to GitHub ===' +Write-Host "Repository : $repoSlug" +Write-Host "Base : $BaseBranch" +Write-Host "Branch : $BranchName" +Write-Host "Resources : $($localizedFiles.Count) file(s) from $sourceResources" +if ($DryRun) { + Write-Host 'Mode : DRY RUN (no push, no pull request changes)' +} +Write-Host '' + +$originalLocation = Get-Location + +try { + #region Clone and stage + + if (Test-Path -LiteralPath $WorkingDirectory) { + Remove-Item -LiteralPath $WorkingDirectory -Recurse -Force + } + + # Authenticate through git's environment-based config rather than the remote + # URL, so the token never lands in .git/config, on a process command line, or + # in Invoke-Git's exception messages. GIT_CONFIG_* is honoured by git 2.31+ + # and applies to clone, fetch and push alike. + $basicAuth = [Convert]::ToBase64String( + [Text.Encoding]::ASCII.GetBytes("x-access-token:$AccessToken")) + $env:GIT_CONFIG_COUNT = '1' + $env:GIT_CONFIG_KEY_0 = 'http.https://github.com/.extraheader' + $env:GIT_CONFIG_VALUE_0 = "AUTHORIZATION: basic $basicAuth" + $cloneUrl = "https://github.com/$repoSlug.git" + + Write-Host "Cloning $repoSlug@$BaseBranch..." + Invoke-Git clone --branch $BaseBranch --quiet $cloneUrl $WorkingDirectory + + Set-Location -LiteralPath $WorkingDirectory + + Invoke-Git config user.email 'sqlclient@microsoft.com' + Invoke-Git config user.name 'SqlClient DevOps' + + $baseSha = (& git rev-parse HEAD).Trim() + Write-Host "Base HEAD : $baseSha" + + # Fetch the automation branch if it already exists so we can tell an actual + # content change apart from a re-run that would produce an identical commit. + & git fetch origin "refs/heads/${BranchName}:refs/remotes/origin/$BranchName" --quiet 2>&1 | Out-Null + $remoteBranchSha = (& git rev-parse --verify --quiet "refs/remotes/origin/$BranchName") + $remoteBranchExists = -not [string]::IsNullOrWhiteSpace($remoteBranchSha) + + if ($remoteBranchExists) { + Write-Host "Branch HEAD: $($remoteBranchSha.Trim())" + } + + $targetResources = Join-Path $WorkingDirectory $ResourcesPath + if (-not (Test-Path -LiteralPath $targetResources)) { + throw "Resources folder '$ResourcesPath' does not exist in $repoSlug@$BaseBranch." + } + + Copy-Item -Path (Join-Path $sourceResources $ResourceFilePattern) -Destination $targetResources -Force + + Invoke-Git add --all -- $ResourcesPath + + & git diff --cached --quiet + if ($LASTEXITCODE -eq 0) { + Write-Host '' + Write-Host "No localization changes relative to '$BaseBranch'. Nothing to publish." + exit 0 + } + + #endregion + + #region Commit and push + + Write-Host '' + Write-Host "Committing localized resources onto '$BranchName'..." + + # Always rebuild the branch from the current base so the pull request stays + # mergeable, rather than accumulating commits run over run. + Invoke-Git checkout -B $BranchName --quiet + Invoke-Git commit --quiet --message $PrTitle + + $newTree = (& git rev-parse 'HEAD^{tree}').Trim() + + $branchUpToDate = $false + if ($remoteBranchExists) { + $remoteTree = (& git rev-parse "refs/remotes/origin/$BranchName^{tree}" 2>$null) + $remoteParent = (& git rev-parse --verify --quiet "refs/remotes/origin/$BranchName^") + + $branchUpToDate = + $null -ne $remoteTree -and + $remoteTree.Trim() -eq $newTree -and + $null -ne $remoteParent -and + $remoteParent.Trim() -eq $baseSha + } + + if ($branchUpToDate) { + Write-Host "Branch '$BranchName' already has these exact changes on top of '$BaseBranch'. Skipping push." + } + elseif ($DryRun) { + Write-Host "[DryRun] Would push '$BranchName' to $repoSlug." + } + else { + # Pin the push to the remote SHA observed above so an overlapping run that + # already updated the branch makes this push fail loudly, instead of + # silently discarding the other run's localization result. + $pushArgs = @('push', 'origin', "${BranchName}:refs/heads/$BranchName", '--quiet') + if ($remoteBranchExists) { + $pushArgs += "--force-with-lease=refs/heads/${BranchName}:$($remoteBranchSha.Trim())" + } + + Invoke-Git @pushArgs + Write-Host "Pushed '$BranchName'." + } + + #endregion + + #region Create or reuse the pull request + + Write-Host '' + Write-Host 'Checking for an existing open pull request...' + + $encodedHead = [Uri]::EscapeDataString("${repoOwner}:$BranchName") + $encodedBase = [Uri]::EscapeDataString($BaseBranch) + $listUri = "https://api.github.com/repos/$repoSlug/pulls?state=open&head=$encodedHead&base=$encodedBase" + + $openPrs = @(Invoke-GitHubApi -Uri $listUri -Headers $headers) + $existingPr = $openPrs | Select-Object -First 1 + + $body = Get-PullRequestBody -Branch $BranchName -UpdatedUtc ([datetime]::UtcNow) + + if ($existingPr) { + $prNumber = $existingPr.number + + if ($DryRun) { + Write-Host "[DryRun] Would refresh open PR #$prNumber ($($existingPr.html_url))." + } + else { + Write-Host "Reusing open PR #$prNumber - refreshing its description." + + $patchUri = "https://api.github.com/repos/$repoSlug/pulls/$prNumber" + Invoke-GitHubApi -Uri $patchUri -Method 'PATCH' -Headers $headers -Body @{ + title = $PrTitle + body = $body + } | Out-Null + + Write-Host "Pull request updated: $($existingPr.html_url)" + } + } + elseif ($DryRun) { + Write-Host "[DryRun] No open pull request found. Would create one ($BranchName -> $BaseBranch)." + } + else { + Write-Host 'No open pull request found. Creating one...' + + $createUri = "https://api.github.com/repos/$repoSlug/pulls" + $newPr = Invoke-GitHubApi -Uri $createUri -Method 'POST' -Headers $headers -Body @{ + title = $PrTitle + head = $BranchName + base = $BaseBranch + body = $body + } + + Write-Host "Pull request created: $($newPr.html_url)" + } + + #endregion + + Write-Host '' + if ($DryRun) { + Write-Host '=== Done (dry run - nothing was published) ===' + } + else { + Write-Host '=== Done ===' + } +} +finally { + Set-Location -LiteralPath $originalLocation + + Remove-Item Env:\GIT_CONFIG_COUNT -ErrorAction SilentlyContinue + Remove-Item Env:\GIT_CONFIG_KEY_0 -ErrorAction SilentlyContinue + Remove-Item Env:\GIT_CONFIG_VALUE_0 -ErrorAction SilentlyContinue + + if ($ownedWorkingDirectory -and (Test-Path -LiteralPath $WorkingDirectory)) { + Remove-Item -LiteralPath $WorkingDirectory -Recurse -Force -ErrorAction SilentlyContinue + } +} diff --git a/eng/pipelines/scripts/tests/Install-DockerCli.macos.Tests.ps1 b/eng/pipelines/scripts/tests/Install-DockerCli.macos.Tests.ps1 new file mode 100644 index 0000000000..08fdcb8076 --- /dev/null +++ b/eng/pipelines/scripts/tests/Install-DockerCli.macos.Tests.ps1 @@ -0,0 +1,194 @@ +<# +.SYNOPSIS + Pester tests for Install-DockerCli.macos.ps1. + +.DESCRIPTION + These cover the bottle selection rules, which are the part that fails by + silently installing the wrong artifact rather than by crashing: picking the + newest version that is actually bottled for Intel, never taking an arm64, + linux or ':all' bottle, and reading a revision build out of the unrevised + cellar directory. + + 'Invoke-RestMethod', 'Invoke-WebRequest' and 'tar' are mocked, so the tests + need no network and no macOS. +#> + +BeforeAll { + $global:scriptPath = Join-Path $PSScriptRoot '..' 'Install-DockerCli.macos.ps1' + + # A stand-in for the bottle archive. The script verifies what it downloads + # against the digest the registry advertises, so the tests have to advertise + # this content's real hash. + $global:blobBytes = [System.Text.Encoding]::UTF8.GetBytes('not-really-a-bottle') + $blobHash = ( + [System.Security.Cryptography.SHA256]::HashData($global:blobBytes) | + ForEach-Object { $_.ToString('x2') } + ) -join '' + $global:blobDigest = "sha256:$blobHash" + + function New-Index { + <# + .SYNOPSIS + An OCI image index annotated with the given bottle ref names. + #> + param([string[]]$RefName) + + return [pscustomobject]@{ + manifests = @( + $RefName | ForEach-Object { + [pscustomobject]@{ + digest = "sha256:digest-$_" + annotations = [pscustomobject]@{ 'org.opencontainers.image.ref.name' = $_ } + } + } + ) + } + } + + function Get-TarMember { + <# + .SYNOPSIS + The archive member the script asked tar to extract. + #> + return @($global:tarArgs)[-1] + } +} + +Describe 'Install-DockerCli.macos.ps1' -Skip:([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture -ne 'X64') { + + BeforeAll { + Mock -CommandName 'Invoke-RestMethod' -MockWith { + if ($Uri -like '*/token?*') { + return [pscustomobject]@{ token = 'test-token' } + } + + if ($Uri -like '*/tags/list*') { + if ($ResponseHeadersVariable) { + Set-Variable -Name $ResponseHeadersVariable -Value @{} -Scope Global + } + return [pscustomobject]@{ tags = $global:tags } + } + + if ($Uri -like '*/manifests/sha256:*') { + $global:manifestRequests += $Uri + return [pscustomobject]@{ + layers = @([pscustomobject]@{ digest = $global:advertisedDigest }) + } + } + + $version = $Uri -replace '.*/manifests/', '' + if (-not $global:refsByVersion.ContainsKey($version)) { + throw "Unexpected manifest request for '$version'." + } + return New-Index -RefName $global:refsByVersion[$version] + } + + Mock -CommandName 'Invoke-WebRequest' -MockWith { + [System.IO.File]::WriteAllBytes($OutFile, $global:blobBytes) + } + + Mock -CommandName 'tar' -MockWith { + $global:tarArgs = $args + $global:LASTEXITCODE = 0 + } + } + + BeforeEach { + $global:tags = @('29.7.2') + $global:refsByVersion = @{ '29.7.2' = @('29.7.2.sonoma') } + $global:advertisedDigest = $global:blobDigest + $global:manifestRequests = @() + $global:tarArgs = @() + + $global:destination = Join-Path ([System.IO.Path]::GetTempPath()) "docker-cli-test-$([guid]::NewGuid().ToString('n'))" + } + + AfterEach { + if (Test-Path -LiteralPath $global:destination) { + Remove-Item -LiteralPath $global:destination -Recurse -Force -ErrorAction SilentlyContinue + } + } + + Context 'Version selection' { + + It 'Takes the newest version that is bottled for Intel' { + $global:tags = @('29.7.2', '29.8.0') + $global:refsByVersion = @{ + '29.8.0' = @('29.8.0.arm64_sequoia', '29.8.0.x86_64_linux') + '29.7.2' = @('29.7.2.sonoma') + } + + & $global:scriptPath -DestinationPath $global:destination + + Get-TarMember | Should -Be 'docker/29.7.2/bin/docker' + } + + It 'Ignores tags that are not versions' { + $global:tags = @('latest', '29.7', '29.7.2-beta', '29.7.2') + + & $global:scriptPath -DestinationPath $global:destination + + Get-TarMember | Should -Be 'docker/29.7.2/bin/docker' + } + + It 'Keeps scanning past a long run of arm64-only versions' { + # Homebrew has stopped publishing Intel bottles, so the newest usable + # version sinks further down the list with every docker release. A + # fixed scan window would eventually stop reaching it. + $global:tags = @('29.7.2') + (0..19 | ForEach-Object { "30.$_.0" }) + $global:refsByVersion = @{ '29.7.2' = @('29.7.2.sonoma') } + foreach ($i in 0..19) { + $global:refsByVersion["30.$i.0"] = @("30.$i.0.arm64_sequoia", "30.$i.0.x86_64_linux") + } + + & $global:scriptPath -DestinationPath $global:destination + + Get-TarMember | Should -Be 'docker/29.7.2/bin/docker' + } + + It 'Ranks a revision build above its base version' { + $global:tags = @('29.7.2', '29.7.2-1') + $global:refsByVersion = @{ + '29.7.2-1' = @('29.7.2.sonoma.1') + '29.7.2' = @('29.7.2.sonoma') + } + + & $global:scriptPath -DestinationPath $global:destination + + # Homebrew keeps the cellar directory at the plain version, so the + # revision must not leak into the member path. + Get-TarMember | Should -Be 'docker/29.7.2/bin/docker' + $global:manifestRequests | Should -Contain 'https://ghcr.io/v2/homebrew/core/docker/manifests/sha256:digest-29.7.2.sonoma.1' + } + } + + Context 'Platform selection' { + + It 'Prefers the newest macOS codename that is bottled' { + $global:refsByVersion = @{ '29.7.2' = @('29.7.2.ventura', '29.7.2.sonoma') } + + & $global:scriptPath -DestinationPath $global:destination + + $global:manifestRequests | Should -Contain 'https://ghcr.io/v2/homebrew/core/docker/manifests/sha256:digest-29.7.2.sonoma' + } + + It 'Never selects an arm64, linux or :all bottle' { + $global:refsByVersion = @{ + '29.7.2' = @('29.7.2.arm64_sonoma', '29.7.2.arm64_linux', '29.7.2.x86_64_linux', '29.7.2.all') + } + + { & $global:scriptPath -DestinationPath $global:destination } | + Should -Throw '*No Intel macOS docker bottle*' + } + } + + Context 'Download integrity' { + + It 'Fails when the bottle does not match the advertised digest' { + $global:advertisedDigest = 'sha256:' + ('0' * 64) + + { & $global:scriptPath -DestinationPath $global:destination } | + Should -Throw '*digest mismatch*' + } + } +} diff --git a/eng/pipelines/scripts/tests/Open-LocalizationPr.Tests.ps1 b/eng/pipelines/scripts/tests/Open-LocalizationPr.Tests.ps1 new file mode 100644 index 0000000000..9f99666c9a --- /dev/null +++ b/eng/pipelines/scripts/tests/Open-LocalizationPr.Tests.ps1 @@ -0,0 +1,357 @@ +<# +.SYNOPSIS + Pester tests for Open-LocalizationPr.ps1. + +.DESCRIPTION + These tests focus on the de-duplication behaviour: a scheduled run must + reuse the existing open localization pull request rather than opening a new + one, and must do nothing at all when there is no localization delta. + + 'git' and 'Invoke-RestMethod' are mocked, so no network or repository + access is required. +#> + +BeforeAll { + $global:scriptPath = Join-Path $PSScriptRoot '..' 'Open-LocalizationPr.ps1' + $global:resourcesPath = 'src/Microsoft.Data.SqlClient/src/Resources' + + function New-TempDirectoryPath { + return Join-Path ([System.IO.Path]::GetTempPath()) "loc-test-$([guid]::NewGuid().ToString('n'))" + } + + function New-SourceTree { + $root = New-TempDirectoryPath + $resources = Join-Path $root $global:resourcesPath + New-Item -ItemType Directory -Path $resources -Force | Out-Null + + foreach ($culture in @('de', 'fr', 'ja')) { + Set-Content -Path (Join-Path $resources "Strings.$culture.resx") -Value '' + } + + return $root + } + + function Get-RestCallCount { + param( + [Parameter(Mandatory)][string]$Method, + [Parameter(Mandatory)][string]$UriPattern + ) + + return @($global:restCalls | Where-Object { $_.Method -eq $Method -and $_.Uri -like $UriPattern }).Count + } + + function Get-RestCallsTo { + param( + [Parameter(Mandatory)][string]$Method, + [Parameter(Mandatory)][string]$UriPattern + ) + + return , @($global:restCalls | Where-Object { $_.Method -eq $Method -and $_.Uri -like $UriPattern }) + } +} + +Describe 'Open-LocalizationPr.ps1' { + + BeforeAll { + Mock -CommandName 'git' -MockWith { + $joined = $args -join ' ' + $global:gitCalls += $joined + $global:LASTEXITCODE = 0 + + if ($joined -like 'clone*') { + # Stand in for a real clone so the script finds the expected layout. + New-Item -ItemType Directory -Force -Path (Join-Path $global:workDir $global:resourcesPath) | Out-Null + return + } + + if ($joined -like 'diff --cached --quiet*') { + $global:LASTEXITCODE = $global:diffExitCode + return + } + + if ($joined -like 'rev-parse HEAD^{tree}*') { return $global:localTree } + if ($joined -like 'rev-parse HEAD*') { return $global:baseSha } + if ($joined -like 'rev-parse *origin/*^{tree}*') { return $global:remoteTree } + if ($joined -like 'rev-parse *origin/*^') { return $global:remoteParent } + if ($joined -like 'rev-parse *origin/*') { return $global:remoteSha } + + return $null + } + + Mock -CommandName 'Invoke-RestMethod' -MockWith { + $global:restCalls += @{ Method = $Method; Uri = $Uri; Body = $Body } + + if ($Method -eq 'GET') { + return $global:openPullRequests + } + + return @{ number = 999; html_url = 'https://github.com/dotnet/SqlClient/pull/999' } + } + } + + BeforeEach { + $global:gitCalls = @() + $global:restCalls = @() + $global:diffExitCode = 1 # by default, there is a localization delta + $global:baseSha = 'base000000' + $global:localTree = 'tree111111' + $global:remoteSha = $null # by default, the branch does not exist yet + $global:remoteTree = $null + $global:remoteParent = $null + $global:openPullRequests = @() + + $global:sourceDir = New-SourceTree + $global:workDir = New-TempDirectoryPath + } + + AfterEach { + foreach ($path in @($global:sourceDir, $global:workDir)) { + if ($path -and (Test-Path -LiteralPath $path)) { + Remove-Item -LiteralPath $path -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + + Context 'Input validation' { + + It 'Requires an access token' { + { & $global:scriptPath -GitHubRepository 'dotnet/SqlClient' -AccessToken '' -SourceDirectory $global:sourceDir } | + Should -Throw '*access token is required*' + } + + It 'Rejects a repository that is not in owner/repo form' { + { & $global:scriptPath -GitHubRepository 'SqlClient' -AccessToken 'token' -SourceDirectory $global:sourceDir } | + Should -Throw "*'owner/repo' form*" + } + + It 'Fails when the localized resources are missing' { + Remove-Item -Path (Join-Path $global:sourceDir $global:resourcesPath 'Strings.*.resx') -Force + + { & $global:scriptPath -GitHubRepository 'dotnet/SqlClient' -AccessToken 'token' -SourceDirectory $global:sourceDir } | + Should -Throw '*Did the OneLocBuild step run?*' + } + } + + Context 'When there is no localization delta' { + + It 'Does not push a branch or touch the GitHub API' { + $global:diffExitCode = 0 + + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $global:restCalls.Count | Should -Be 0 + @($global:gitCalls | Where-Object { $_ -like 'push*' }).Count | Should -Be 0 + @($global:gitCalls | Where-Object { $_ -like 'commit*' }).Count | Should -Be 0 + } + } + + Context 'When a localization pull request is already open' { + + BeforeEach { + $global:openPullRequests = @( + @{ number = 4612; html_url = 'https://github.com/dotnet/SqlClient/pull/4612' } + ) + } + + It 'Updates the existing pull request instead of opening another one' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + (Get-RestCallCount -Method 'POST' -UriPattern '*/pulls') | Should -Be 0 + (Get-RestCallCount -Method 'PATCH' -UriPattern '*/pulls/4612') | Should -Be 1 + } + + It 'Looks the pull request up by the stable head branch' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $lookups = Get-RestCallsTo -Method 'GET' -UriPattern '*/pulls?*' + $lookups.Count | Should -Be 1 + $lookups[0].Uri | Should -BeLike '*state=open*' + $lookups[0].Uri | Should -BeLike '*dotnet%3Adev%2Fautomation%2Fonelocbuild*' + } + } + + Context 'When no localization pull request is open' { + + It 'Creates exactly one pull request' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + (Get-RestCallCount -Method 'POST' -UriPattern '*/repos/dotnet/SqlClient/pulls') | Should -Be 1 + } + + It 'Pushes a stable branch name rather than a timestamped one' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $push = @($global:gitCalls | Where-Object { $_ -like 'push*' }) + $push.Count | Should -Be 1 + $push[0] | Should -BeLike '*dev/automation/onelocbuild:refs/heads/dev/automation/onelocbuild*' + $push[0] | Should -Not -Match 'onelocbuild-\d' + } + + It 'Normalizes a repository value that carries a .git suffix' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient.git' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + (Get-RestCallCount -Method 'POST' -UriPattern 'https://api.github.com/repos/dotnet/SqlClient/pulls') | + Should -Be 1 + } + } + + Context 'When the branch already carries the identical change' { + + BeforeEach { + $global:remoteSha = 'remote0000' + $global:remoteTree = $global:localTree + $global:remoteParent = $global:baseSha + $global:openPullRequests = @( + @{ number = 4612; html_url = 'https://github.com/dotnet/SqlClient/pull/4612' } + ) + } + + It 'Skips the force-push but still keeps the pull request current' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + @($global:gitCalls | Where-Object { $_ -like 'push*' }).Count | Should -Be 0 + (Get-RestCallCount -Method 'PATCH' -UriPattern '*/pulls/4612') | Should -Be 1 + (Get-RestCallCount -Method 'POST' -UriPattern '*/pulls') | Should -Be 0 + } + } + + Context 'When pushing over an existing remote branch' { + + BeforeEach { + $global:remoteSha = 'remote0000' + $global:remoteTree = 'differenttree' + $global:remoteParent = 'staleparent' + $global:openPullRequests = @( + @{ number = 4612; html_url = 'https://github.com/dotnet/SqlClient/pull/4612' } + ) + } + + It 'Leases the push against the observed remote SHA' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $push = @($global:gitCalls | Where-Object { $_ -like 'push*' }) + $push.Count | Should -Be 1 + $push[0] | Should -BeLike '*--force-with-lease=refs/heads/dev/automation/onelocbuild:remote0000*' + $push[0] | Should -Not -Match '(^|\s)--force(\s|$)' + } + } + + Context 'When the remote branch does not exist yet' { + + It 'Pushes without forcing' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $push = @($global:gitCalls | Where-Object { $_ -like 'push*' }) + $push.Count | Should -Be 1 + $push[0] | Should -Not -Match '--force' + } + } + + Context 'Credential handling' { + + It 'Keeps the access token out of the git remote URL' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'super-secret-token-value' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $global:gitCalls | Should -Not -Match 'super-secret-token-value' + @($global:gitCalls | Where-Object { $_ -like 'clone*' })[0] | + Should -BeLike '*https://github.com/dotnet/SqlClient.git*' + } + + It 'Clears the git credential environment afterwards' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir + + $env:GIT_CONFIG_COUNT | Should -BeNullOrEmpty + $env:GIT_CONFIG_VALUE_0 | Should -BeNullOrEmpty + } + } + + Context 'When running in dry-run mode' { + + It 'Neither pushes nor creates a pull request' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir ` + -DryRun + + @($global:gitCalls | Where-Object { $_ -like 'push*' }).Count | Should -Be 0 + (Get-RestCallCount -Method 'POST' -UriPattern '*/pulls') | Should -Be 0 + (Get-RestCallCount -Method 'PATCH' -UriPattern '*/pulls/*') | Should -Be 0 + } + + It 'Still performs the existing pull request lookup' { + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir ` + -DryRun + + (Get-RestCallCount -Method 'GET' -UriPattern '*/pulls?*') | Should -Be 1 + } + + It 'Does not update an existing pull request' { + $global:remoteSha = 'remote0000' + $global:remoteTree = 'differenttree' + $global:remoteParent = 'oldbase000' + $global:openPullRequests = @( + @{ number = 4612; html_url = 'https://github.com/dotnet/SqlClient/pull/4612' } + ) + + & $global:scriptPath ` + -GitHubRepository 'dotnet/SqlClient' ` + -AccessToken 'token' ` + -SourceDirectory $global:sourceDir ` + -WorkingDirectory $global:workDir ` + -DryRun + + @($global:gitCalls | Where-Object { $_ -like 'push*' }).Count | Should -Be 0 + (Get-RestCallCount -Method 'PATCH' -UriPattern '*/pulls/4612') | Should -Be 0 + } + } +} diff --git a/eng/pipelines/scripts/tests/README.md b/eng/pipelines/scripts/tests/README.md new file mode 100644 index 0000000000..70534a9ed1 --- /dev/null +++ b/eng/pipelines/scripts/tests/README.md @@ -0,0 +1,30 @@ +# Pipeline Script Tests + +Pester tests for the PowerShell helpers under `eng/pipelines/scripts/`. + +## Prerequisites + +These tests require **Pester v5 or later**: + +```powershell +Install-Module Pester -MinimumVersion 5.0 -Scope CurrentUser -Force -SkipPublisherCheck +``` + +## Running the tests + +```powershell +Import-Module Pester -MinimumVersion 5.0 +Invoke-Pester ./eng/pipelines/scripts/tests/ +``` + +Add `-Output Detailed` to see per-test results. + +## Test files + +| File | Covers | +| ---- | ------ | +| `Open-LocalizationPr.Tests.ps1` | `Open-LocalizationPr.ps1` — de-duplication of the scheduled localization pull request. | +| `Install-DockerCli.macos.Tests.ps1` | `Install-DockerCli.macos.ps1` — Homebrew bottle selection for the macOS docker CLI. | + +`git`, `tar`, `Invoke-RestMethod` and `Invoke-WebRequest` are mocked, so the +tests never touch the network or a real repository. diff --git a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml index cb25fe1f4e..f4427f82e8 100644 --- a/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-package-ref-pipeline.yml @@ -34,9 +34,11 @@ pr: branches: include: # GitHub repo branch targets that will trigger PR validation builds. - - dev/* - - feat/* - - main + # + # GOTCHA: Currently disabled on all but release/* due to limited resources. + #- dev/* + #- feat/* + #- main - release/* paths: @@ -135,11 +137,19 @@ extends: debug: ${{ parameters.debug }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + # Keep the primary (SQL 2025 / Azure SQL) configurations on the same set of + # target frameworks as everything else. Only the CI-SqlClient pipeline + # opts in to the broader .NET 10.0 coverage on those configurations. + primaryTargetFrameworks: ${{ parameters.targetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} # Legacy SQL Server tests (2016/2017) run in CI only, not on PRs. runLegacySqlTests: false + # SQL Server 2022 coverage runs in CI only. PR validation covers SQL + # Server 2019, 2025 and Azure SQL. + runSql22Tests: false # Don't run the AE tests in Debug mode; they rarely succeed. ${{ if eq(parameters.buildConfiguration, 'Debug') }}: runAlwaysEncryptedTests: false diff --git a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml index b2a5edf4e4..74309ae643 100644 --- a/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml +++ b/eng/pipelines/sqlclient-pr-project-ref-pipeline.yml @@ -135,11 +135,19 @@ extends: debug: ${{ parameters.debug }} targetFrameworks: ${{ parameters.targetFrameworks }} targetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} + # Keep the primary (SQL 2025 / Azure SQL) configurations on the same set of + # target frameworks as everything else. Only the CI-SqlClient pipeline + # opts in to the broader .NET 10.0 coverage on those configurations. + primaryTargetFrameworks: ${{ parameters.targetFrameworks }} + primaryTargetFrameworksUnix: ${{ parameters.targetFrameworksUnix }} testJobTimeout: ${{ parameters.testJobTimeout }} testSets: ${{ parameters.testSets }} useManagedSNI: ${{ parameters.useManagedSNI }} # Legacy SQL Server tests (2016/2017) run in CI only, not on PRs. runLegacySqlTests: false + # SQL Server 2022 coverage runs in CI only. PR validation covers SQL + # Server 2019, 2025 and Azure SQL. + runSql22Tests: false # Don't run the AE tests in Debug mode; they rarely succeed. ${{ if eq(parameters.buildConfiguration, 'Debug') }}: runAlwaysEncryptedTests: false diff --git a/eng/pipelines/stages/build-abstractions-package-ci-stage.yml b/eng/pipelines/stages/build-abstractions-package-ci-stage.yml index 3df0ef9843..03fac7ae8e 100644 --- a/eng/pipelines/stages/build-abstractions-package-ci-stage.yml +++ b/eng/pipelines/stages/build-abstractions-package-ci-stage.yml @@ -35,6 +35,19 @@ parameters: type: object default: [] + # The name of the pool to use for jobs that require customized VM images. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + # Any pool specified here must contain images with the following names: + # + # - ADO-Win25 + # - ADO-UB24 + # + - name: poolName + type: string + # The type of build to produce (Release or Debug) - name: buildConfiguration type: string @@ -105,8 +118,8 @@ stages: jobNameSuffix: linux netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] - poolName: Azure Pipelines - vmImage: ubuntu-latest + poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 # ------------------------------------------------------------------------ # Build and test on Windows @@ -120,8 +133,8 @@ stages: jobNameSuffix: windows netFrameworkRuntimes: [net462] netRuntimes: [net8.0, net9.0, net10.0] - poolName: Azure Pipelines - vmImage: windows-latest + poolName: ${{ parameters.poolName }} + poolImage: ADO-Win25 # ------------------------------------------------------------------------ # Build and test on macOS. @@ -135,14 +148,18 @@ stages: jobNameSuffix: macos netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] + # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted + # 'Azure Pipelines' pool. poolName: Azure Pipelines - vmImage: macos-latest + poolImage: macos-latest # ------------------------------------------------------------------------ # Create and publish the NuGet package. - template: /eng/pipelines/jobs/pack-abstractions-package-ci-job.yml@self parameters: + poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} # The version is computed by this stage (see the packageVersion variable above). packageVersion: $(packageVersion) diff --git a/eng/pipelines/stages/build-azure-package-ci-stage.yml b/eng/pipelines/stages/build-azure-package-ci-stage.yml index 0713030b76..1f502a5fee 100644 --- a/eng/pipelines/stages/build-azure-package-ci-stage.yml +++ b/eng/pipelines/stages/build-azure-package-ci-stage.yml @@ -40,29 +40,23 @@ parameters: default: [] # The name of the pool to use for jobs that require customized VM images. - - name: adoPoolName + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + # Any pool specified here must contain images with the following names: + # + # - ADO-MMS25-SQL25 + # - ADO-UB24-SQL25 + # + - name: poolName type: string - # This variable should be defined in AzureDevOps Library variable groups, - # for both the Public and ADO.Net projects. - # - # Any pool specified here must contain images with the following names: - # - # - ADO-MMS22-SQL22 - # - ADO-UB22-SQL22 - # - default: $(ci_var_defaultPoolName) # The name of the pipeline artifacts to publish. - name: azureArtifactsName type: string default: Azure.Artifacts - # The name of the general Azure pool to use for jobs that don't require - # customized VM images. - - name: azurePoolName - type: string - default: Azure Pipelines - # The type of build to produce (Release or Debug) - name: buildConfiguration type: string @@ -144,25 +138,6 @@ stages: # ------------------------------------------------------------------------ # Build and test on Linux. - - template: /eng/pipelines/jobs/test-azure-package-ci-job.yml@self - parameters: - abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} - packageVersion: $(packageVersion) - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayNamePrefix: Linux - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - jobNameSuffix: linux - loggingArtifactsName: ${{ parameters.loggingArtifactsName }} - mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} - sqlServerPackageVersion: $(sqlServerPackageVersion) - netFrameworkRuntimes: [] - netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.azurePoolName }} - referenceType: ${{ parameters.referenceType }} - vmImage: ubuntu-latest - # Use our 1ES ADO pool for comprehensive testing. - template: /eng/pipelines/jobs/test-azure-package-ci-job.yml@self parameters: @@ -179,7 +154,7 @@ stages: sqlServerPackageVersion: $(sqlServerPackageVersion) netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.adoPoolName }} + poolName: ${{ parameters.poolName }} referenceType: ${{ parameters.referenceType }} saPassword: $(saPassword) # The image includes a SQL Server instance that we must configure. @@ -187,31 +162,11 @@ stages: - template: /eng/pipelines/common/templates/steps/configure-sql-server-linux-step.yml@self parameters: saPassword: $(saPassword) - vmImage: ADO-UB22-SQL22 + poolImage: ADO-UB24-SQL25 # ------------------------------------------------------------------------ # Build and test on Windows - # Use the Azure Pipelines pool for basic testing. - - template: /eng/pipelines/jobs/test-azure-package-ci-job.yml@self - parameters: - abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} - packageVersion: $(packageVersion) - buildConfiguration: ${{ parameters.buildConfiguration }} - debug: ${{ parameters.debug }} - displayNamePrefix: Win - dotnetVerbosity: ${{ parameters.dotnetVerbosity }} - jobNameSuffix: windows - loggingArtifactsName: ${{ parameters.loggingArtifactsName }} - mdsArtifactsName: ${{ parameters.mdsArtifactsName }} - sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} - sqlServerPackageVersion: $(sqlServerPackageVersion) - netFrameworkRuntimes: [net462] - netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.azurePoolName }} - referenceType: ${{ parameters.referenceType }} - vmImage: windows-latest - # Use our 1ES ADO pool for comprehensive testing. - template: /eng/pipelines/jobs/test-azure-package-ci-job.yml@self parameters: @@ -228,7 +183,7 @@ stages: sqlServerPackageVersion: $(sqlServerPackageVersion) netFrameworkRuntimes: [net462] netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.adoPoolName }} + poolName: ${{ parameters.poolName }} referenceType: ${{ parameters.referenceType }} saPassword: $(saPassword) # The image includes a SQL Server instance that we must configure. @@ -241,11 +196,8 @@ stages: # These variables are from an Azure DevOps Library variable # group. fileStreamDirectory: $(FileStreamDirectory) - sqlRootPath: $(SQL22RootPath) - # The ADO-MMS22-SQL22 image includes a local SQL Server that supports - # integrated security. - supportsIntegratedSecurity: true - vmImage: ADO-MMS22-SQL22 + SQLRootPath: $(SQL25RootPath) + poolImage: ADO-MMS25-SQL25 # ------------------------------------------------------------------------ # Build and test on macOS. @@ -266,17 +218,19 @@ stages: sqlServerPackageVersion: $(sqlServerPackageVersion) netFrameworkRuntimes: [] netRuntimes: [net8.0, net9.0, net10.0] - poolName: ${{ parameters.azurePoolName }} + # Our 1ES pools do not offer macOS images, so this job runs on the Microsoft-hosted + # 'Azure Pipelines' pool. + poolName: Azure Pipelines referenceType: ${{ parameters.referenceType }} - vmImage: macos-latest - - # We do not currently have any images in our 1ES ADO pools for macOS. + poolImage: macos-latest # ------------------------------------------------------------------------ # Create and publish the NuGet package. - template: /eng/pipelines/jobs/pack-azure-package-ci-job.yml@self parameters: + poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} packageVersion: $(packageVersion) azureArtifactsName: ${{ parameters.azureArtifactsName }} @@ -285,9 +239,7 @@ stages: dependsOn: # We depend on all of the test jobs to ensure the tests pass before # producing the NuGet package. - - test_azure_package_job_linux - test_azure_package_job_linux_integration - - test_azure_package_job_windows - test_azure_package_job_windows_integration - test_azure_package_job_macos dotnetVerbosity: ${{ parameters.dotnetVerbosity }} diff --git a/eng/pipelines/stages/build-logging-package-ci-stage.yml b/eng/pipelines/stages/build-logging-package-ci-stage.yml index b9d5feb082..62b52480c9 100644 --- a/eng/pipelines/stages/build-logging-package-ci-stage.yml +++ b/eng/pipelines/stages/build-logging-package-ci-stage.yml @@ -25,6 +25,14 @@ parameters: + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + # Additional stages we depend on, if any. - name: additionalDependsOn type: object @@ -84,6 +92,8 @@ stages: - template: /eng/pipelines/jobs/pack-logging-package-ci-job.yml@self parameters: + poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 loggingArtifactsName: ${{ parameters.loggingArtifactsName }} # The version is computed by this stage (see the packageVersion variable above). packageVersion: $(packageVersion) diff --git a/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml b/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml index 70fa122d5f..f7eb9d6f1a 100644 --- a/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml +++ b/eng/pipelines/stages/build-sqlclient-package-ci-stage.yml @@ -18,6 +18,14 @@ parameters: type: string default: Abstractions.Artifacts + # The name of the 1ES pool to run in. Hosted Azure Pipelines pools are not supported. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + # Additional stages we depend on, if any. - name: additionalDependsOn type: object @@ -84,6 +92,8 @@ stages: jobs: - template: /eng/pipelines/common/templates/jobs/ci-build-nugets-job.yml@self parameters: + poolName: ${{ parameters.poolName }} + poolImage: ADO-Win25 buildConfiguration: ${{ parameters.buildConfiguration }} referenceType: ${{ parameters.referenceType }} abstractionsArtifactsName: ${{ parameters.abstractionsArtifactsName }} diff --git a/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml b/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml index 3a1417185d..f62a6eb166 100644 --- a/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml +++ b/eng/pipelines/stages/build-sqlserver-package-ci-stage.yml @@ -25,6 +25,14 @@ parameters: + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + # Additional stages we depend on, if any. - name: additionalDependsOn type: object @@ -77,6 +85,8 @@ stages: jobs: - template: /eng/pipelines/jobs/pack-sqlserver-package-ci-job.yml@self parameters: + poolName: ${{ parameters.poolName }} + poolImage: ADO-UB24 buildConfiguration: ${{ parameters.buildConfiguration }} debug: ${{ parameters.debug }} sqlServerArtifactsName: ${{ parameters.sqlServerArtifactsName }} diff --git a/eng/pipelines/stages/compute-versions-ci-stage.yml b/eng/pipelines/stages/compute-versions-ci-stage.yml index 9171582cd2..ada6a8844b 100644 --- a/eng/pipelines/stages/compute-versions-ci-stage.yml +++ b/eng/pipelines/stages/compute-versions-ci-stage.yml @@ -19,6 +19,18 @@ # This stage MUST run before all build stages so they can consume computed versions. parameters: + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + # Build suffix appended to prerelease tag (e.g. 'ci' or 'pr'). - name: buildSuffix type: string @@ -32,7 +44,9 @@ stages: - job: compute_versions_job displayName: "Extract versions from Versions.props" pool: - vmImage: ubuntu-latest + name: ${{ parameters.poolName }} + demands: + - imageOverride -equals ${{ parameters.poolImage }} steps: # Install the global.json-pinned .NET SDK before running any dotnet build, so this stage diff --git a/eng/pipelines/stages/generate-secrets-ci-stage.yml b/eng/pipelines/stages/generate-secrets-ci-stage.yml index eaf61e1f60..5e54a5d1b1 100644 --- a/eng/pipelines/stages/generate-secrets-ci-stage.yml +++ b/eng/pipelines/stages/generate-secrets-ci-stage.yml @@ -31,6 +31,18 @@ parameters: type: boolean default: false + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + stages: # The stage downstream stages must depend on to ensure the secrets are generated before they are @@ -48,9 +60,11 @@ stages: - job: secrets_job displayName: Generate Secrets pool: - # We don't need anything special, so use the standard Microsoft-hosted Ubuntu image, which - # is typically very fast to spin up. - vmImage: ubuntu-latest + # This job has no special image requirements, so use a minimal Linux image. + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} steps: diff --git a/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml b/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml index 3dccd61989..d16b4a5838 100644 --- a/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml +++ b/eng/pipelines/stages/verify-nuget-packages-ci-stage.yml @@ -38,6 +38,18 @@ parameters: type: string default: SqlServer.Artifacts + # The name of the 1ES pool to run in. + # + # Supplied by the caller so that the pool name flows down from the pipeline + # root, rather than being read from a variable group at this depth. + # + - name: poolName + type: string + + # The name of the VM image to run on, within the pool. + - name: poolImage + type: string + stages: - stage: verify_nuget_packages_stage @@ -55,8 +67,10 @@ stages: displayName: Verify NuGet Package Metadata pool: - name: Azure Pipelines - vmImage: windows-latest + name: ${{ parameters.poolName }} + + demands: + - imageOverride -equals ${{ parameters.poolImage }} variables: # The directory where all package artifacts will be downloaded. diff --git a/global.json b/global.json index 650a5c0ddf..340837b5af 100644 --- a/global.json +++ b/global.json @@ -2,9 +2,9 @@ "sdk": { // We currently require the .NET 10 SDK to build and test the project. - "version": "10.0.300", + "version": "10.0.401", - // Allow roll-forward within the 10.0.3xx feature band so servicing patches + // Allow roll-forward within the 10.0.4xx feature band so servicing patches // are picked up automatically without requiring a PR for each bump. "rollForward": "patch", diff --git a/release-notes/6.1/6.1.7.md b/release-notes/6.1/6.1.7.md new file mode 100644 index 0000000000..6c46dc8b19 --- /dev/null +++ b/release-notes/6.1/6.1.7.md @@ -0,0 +1,103 @@ +# Release Notes + +## Stable Release 6.1.7 - 2026-09-10 + +This update brings the following changes since the [6.1.6](6.1.6.md) release: + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4598](https://github.com/dotnet/SqlClient/pull/4598)) + +### Fixed + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4584](https://github.com/dotnet/SqlClient/pull/4584)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4552](https://github.com/dotnet/SqlClient/pull/4552)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed token authentication state handling so clearing `SqlConnection.AccessToken` preserves an existing `AccessTokenCallback` in the connection pool key, and clearing `AccessTokenCallback` preserves an existing `AccessToken`. Callback-based authentication now also follows the same prelogin server-certificate validation rules as an explicitly supplied access token. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4560](https://github.com/dotnet/SqlClient/pull/4560)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4664](https://github.com/dotnet/SqlClient/pull/4664)) + +## Target Platform Support + +- .NET Framework 4.6.2+ (Windows x86, Windows x64, Windows ARM64) +- .NET 8.0+ (Windows x86, Windows x64, Windows ARM64, Linux, macOS) +- .NET Standard 2.0+ (Windows x86, Windows x64, Windows ARM64, Linux, macOS) + +### Dependencies + +#### .NET Framework 4.6.2 + +- Azure.Core 1.50.0 +- Azure.Identity 1.17.1 +- Microsoft.Data.SqlClient.SNI 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 +- Microsoft.IdentityModel.JsonWebTokens 7.7.1 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 7.7.1 +- System.Buffers 4.6.1 +- System.Data.Common 4.3.0 +- System.Diagnostics.DiagnosticSource 8.0.1 +- System.IdentityModel.Tokens.Jwt 7.7.1 +- System.Memory 4.6.3 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 8.0.6 +- System.Text.RegularExpressions 4.3.1 + +#### .NET 8.0 + +- Azure.Core 1.50.0 +- Azure.Identity 1.17.1 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 +- Microsoft.IdentityModel.JsonWebTokens 7.7.1 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 7.7.1 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Diagnostics.DiagnosticSource 8.0.1 +- System.IdentityModel.Tokens.Jwt 7.7.1 +- System.Security.Cryptography.Pkcs 8.0.1 + +#### .NET 9.0 + +- Azure.Core 1.50.0 +- Azure.Identity 1.17.1 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 9.0.11 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 +- Microsoft.IdentityModel.JsonWebTokens 7.7.1 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 7.7.1 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 9.0.11 +- System.Diagnostics.DiagnosticSource 9.0.11 +- System.IdentityModel.Tokens.Jwt 7.7.1 +- System.Security.Cryptography.Pkcs 9.0.11 + +#### .NET Standard 2.0 + +- Azure.Core 1.50.0 +- Azure.Identity 1.17.1 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 +- Microsoft.IdentityModel.JsonWebTokens 7.7.1 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 7.7.1 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Diagnostics.DiagnosticSource 8.0.1 +- System.IdentityModel.Tokens.Jwt 7.7.1 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 8.0.6 diff --git a/release-notes/6.1/README.md b/release-notes/6.1/README.md index e119d5baf1..b0e4a4be80 100644 --- a/release-notes/6.1/README.md +++ b/release-notes/6.1/README.md @@ -4,6 +4,7 @@ The following Microsoft.Data.SqlClient 6.1 stable releases have been shipped: | Release Date | Version | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 6.1.7 | [Release Notes](6.1.7.md) | | 2026-06-24 | 6.1.6 | [Release Notes](6.1.6.md) | | 2026-04-27 | 6.1.5 | [Release Notes](6.1.5.md) | | 2026-01-15 | 6.1.4 | [Release Notes](6.1.4.md) | diff --git a/release-notes/7.0/7.0.3.md b/release-notes/7.0/7.0.3.md new file mode 100644 index 0000000000..01dfdc6c03 --- /dev/null +++ b/release-notes/7.0/7.0.3.md @@ -0,0 +1,115 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +This update brings the following changes since the [7.0.2](7.0.2.md) release: + +The core driver and its companion packages ship together as version `7.0.3`. Update the companion packages you use alongside the driver to `7.0.3`. Assembly versions remain `7.0.0.0`, unchanged from `7.0.2`. + +### Companion package release notes + +- [Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.0.3](../add-ons/AzureKeyVaultProvider/7.0/7.0.3.md) +- [Microsoft.Data.SqlClient.Extensions.Azure 7.0.3](../Extensions/Azure/7.0/7.0.3.md) — includes the Entra ID authority parsing fix for Dataverse/Dynamics 365 connections. +- [Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3](../Extensions/Abstractions/7.0/7.0.3.md) +- [Microsoft.Data.SqlClient.Internal.Logging 7.0.3](../Internal/Logging/7.0/7.0.3.md) + +### Changed + +- Updated the `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` dependencies to 6.0.3 (was 6.0.2). + ([#4599](https://github.com/dotnet/SqlClient/pull/4599)) + +### Fixed + +- Fixed a `SqlBulkCopy` regression in environments where the application login cannot read `sys.all_columns`. Bulk copy now falls back to the earlier column-discovery behavior when that permission is unavailable. Support for hidden columns and SQL Graph column aliases still requires access to the metadata view. + ([#4370](https://github.com/dotnet/SqlClient/issues/4370), [#4306](https://github.com/dotnet/SqlClient/pull/4306), [#4402](https://github.com/dotnet/SqlClient/pull/4402)) + +- Fixed a memory-allocation regression in connection and command operations caused by formatting diagnostic strings even when tracing was disabled. Also corrected trace messages that reported an incorrect object ID or could throw `FormatException` when traced values contained braces. + ([#4528](https://github.com/dotnet/SqlClient/pull/4528), [#4533](https://github.com/dotnet/SqlClient/pull/4533)) + +- Fixed `ServerCertificate` validation on the managed SNI path so the configured certificate is compared against the server certificate even when the server certificate passes chain and host-name validation. When certificate validation is enabled, a missing, unreadable, or invalid certificate file, a certificate mismatch, or a missing server certificate now causes the TLS handshake to fail instead of bypassing the configured certificate check. (net8.0/net9.0 only) + ([#4445](https://github.com/dotnet/SqlClient/pull/4445), [#4583](https://github.com/dotnet/SqlClient/pull/4583)) + +- Fixed Always Encrypted VSM/HGS enclave attestation to verify that the enclave public key used to establish a session matches the key committed to by the signed attestation report. Missing, malformed, or mismatched key-binding data now causes attestation to fail before the session secret is derived. + ([#4532](https://github.com/dotnet/SqlClient/pull/4532), [#4553](https://github.com/dotnet/SqlClient/pull/4553)) + +- Fixed `SqlConnection.AccessTokenCallback` not disabling Transparent Network IP Resolution by default, making it consistent with `SqlConnection.AccessToken`. An explicitly configured `TransparentNetworkIPResolution` connection-string value still takes precedence. (net462 only) + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561)) + +- Fixed authentication state handling so clearing `SqlConnection.AccessToken`, `AccessTokenCallback`, or `SspiContextProvider` preserves the other authentication values in the connection pool key. Cloning a connection or updating its credential also preserves its `SspiContextProvider`. Combining a non-null `SspiContextProvider` with `AccessToken` or `AccessTokenCallback` now throws `InvalidOperationException` instead of silently discarding authentication state; applications must use one authentication mechanism at a time. + ([#4520](https://github.com/dotnet/SqlClient/pull/4520), [#4561](https://github.com/dotnet/SqlClient/pull/4561), [#4644](https://github.com/dotnet/SqlClient/pull/4644)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now active only while an explicitly configured custom retry provider is resolved and constructed, and probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-provided handler. (net8.0/net9.0 only) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547), [#4663](https://github.com/dotnet/SqlClient/pull/4663)) + +## Contributors + +We thank the following public contributors. Their efforts toward this project are very much appreciated. + +- [edwardneal](https://github.com/edwardneal) + +## Target Platform Support + +- .NET Framework 4.6.2+ (Windows x86, Windows x64, Windows ARM64) +- .NET 8.0+ (Windows x86, Windows x64, Windows ARM, Windows ARM64, Linux, macOS) + +### Dependencies + +#### .NET 9.0 + +- Microsoft.Bcl.Cryptography 9.0.13 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 9.0.13 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 9.0.13 +- System.Security.Cryptography.Pkcs 9.0.13 + +#### .NET 8.0 + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Security.Cryptography.Pkcs 8.0.1 + +#### .NET Standard 2.0 + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Data.SqlClient.SNI.runtime 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 10.0.3 +- System.Threading.Channels 10.0.3 + +#### .NET Framework 4.6.2+ + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Data.SqlClient.SNI 6.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- System.Buffers 4.6.1 +- System.Data.Common 4.3.0 +- System.Diagnostics.DiagnosticSource 10.0.3 +- System.Memory 4.6.3 +- System.Runtime.InteropServices.RuntimeInformation 4.3.0 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 10.0.3 +- System.Threading.Channels 10.0.3 +- System.ValueTuple 4.6.2 diff --git a/release-notes/7.0/README.md b/release-notes/7.0/README.md index aad84057e1..3057122559 100644 --- a/release-notes/7.0/README.md +++ b/release-notes/7.0/README.md @@ -4,6 +4,7 @@ The following Microsoft.Data.SqlClient 7.0 releases have been shipped: | Release Date | Version | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | | 2026-04-23 | 7.0.1 | [Release Notes](7.0.1.md) | | 2026-03-17 | 7.0.0 | [Release Notes](7.0.0.md) | diff --git a/release-notes/7.1/7.1.0-preview2.md b/release-notes/7.1/7.1.0-preview2.md index 5f4a65bb78..df45fe0d7a 100644 --- a/release-notes/7.1/7.1.0-preview2.md +++ b/release-notes/7.1/7.1.0-preview2.md @@ -108,7 +108,7 @@ This update brings the following changes since the [7.1.0-preview1](7.1.0-previe - Reduced allocations by avoiding lock acquisition on `SqlErrorCollection` counters when no errors exist, and by avoiding stack-trace materialization for expected `null`-return paths. ([#4157](https://github.com/dotnet/SqlClient/pull/4157), [#4099](https://github.com/dotnet/SqlClient/pull/4099), [#4102](https://github.com/dotnet/SqlClient/pull/4102)) - Improved `EnclaveDiffieHellmanInfo.Size` accuracy. ([#4346](https://github.com/dotnet/SqlClient/pull/4346)) - `SqlVector` now serializes and deserializes little-endian multibyte values explicitly for consistent behavior across architectures. ([#3861](https://github.com/dotnet/SqlClient/pull/3861)) -- Updated the bundled .NET 10 SDK to `10.0.300`. ([#4287](https://github.com/dotnet/SqlClient/pull/4287)) +- Updated the .NET 10 SDK used to build the repository to `10.0.300`. ([#4287](https://github.com/dotnet/SqlClient/pull/4287)) ### Fixed diff --git a/release-notes/7.1/7.1.0.md b/release-notes/7.1/7.1.0.md new file mode 100644 index 0000000000..c955e6c5c9 --- /dev/null +++ b/release-notes/7.1/7.1.0.md @@ -0,0 +1,435 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +This is the general availability release of **Microsoft.Data.SqlClient 7.1**. It closes out the `7.1` preview cycle with application identity reporting for telemetry, the deprecation of `TransparentNetworkIPResolution`, and a set of connection, transaction, and Named Pipes fixes. + +> **Important — package version alignment:** Starting with the [7.0.2](../7.0/7.0.2.md) release, the `Microsoft.Data.SqlClient` driver and its companion packages share a single aligned version. The `7.1.0` GA release continues this alignment; the following packages ship together as `7.1.0`: +> +> - `Microsoft.Data.SqlClient` +> - `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` +> - `Microsoft.Data.SqlClient.Extensions.Azure` +> - `Microsoft.Data.SqlClient.Extensions.Abstractions` +> - `Microsoft.Data.SqlClient.Internal.Logging` +> +> (`Microsoft.SqlServer.Server` continues to version independently and remains at `1.0.0`.) +> +> Applications must reference the same versions of `Microsoft.Data.SqlClient` and its extensions for best compatibility. In particular, applications that reference `Microsoft.Data.SqlClient.Extensions.Azure` must upgrade it to `7.1.0` when upgrading `Microsoft.Data.SqlClient` to `7.1.0`. +> +> **Compatibility guarantee:** All aligned assemblies ship with `FileVersion 7.1.0.x` and `AssemblyVersion 7.0.0.0`. The `AssemblyVersion` is unchanged from [7.0.2](../7.0/7.0.2.md), so upgrading from `7.0.2`, `7.0.3`, or any `7.1` preview to `7.1.0` does **not** require any new .NET Framework strong-name binding redirects. Applications upgrading from `7.0.0` or `7.0.1` should note that `Extensions.Azure`, `Extensions.Abstractions`, and `Internal.Logging` raised their `AssemblyVersion` from `1.0.0.0` to `7.0.0.0` in [7.0.2](../7.0/7.0.2.md); see those release notes for the one-time .NET Framework impact. + +### Companion package release notes + +- [Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.1.0](../add-ons/AzureKeyVaultProvider/7.1/7.1.0.md) +- [Microsoft.Data.SqlClient.Extensions.Azure 7.1.0](../Extensions/Azure/7.1/7.1.0.md) +- [Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0](../Extensions/Abstractions/7.1/7.1.0.md) +- [Microsoft.Data.SqlClient.Internal.Logging 7.1.0](../Internal/Logging/7.1/7.1.0.md) + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +### Added + +#### Application Identity in the USERAGENT Payload + +*What Changed:* + +- Added a `RegisteredApplication` enum and a matching `SqlConnection.RegisteredApplication` property that let a library or tool identify itself to SQL Server through version 2 of the TDS USERAGENT feature extension. The payload also carries a new driver-owned 64-bit *Driver Properties* flag field; bit 0 reports whether connection pool V2 is enabled for the process. Both fields are emitted as unpadded uppercase hexadecimal. + ([#3201](https://github.com/dotnet/SqlClient/issues/3201), [#4632](https://github.com/dotnet/SqlClient/pull/4632)) + +*Who Benefits:* + +- Middleware and tooling built on top of the driver — Entity Framework Core, Semantic Kernel, SQL Server Management Studio, SqlPackage, Data API Builder, and similar — can be distinguished in server-side telemetry without the driver accepting arbitrary user-supplied user-agent text. This originated as a request from the Entity Framework Core team. +- Service operators gain a more accurate picture of which client stacks are connecting, which helps when diagnosing workload-specific behavior. + +*Impact:* + +- Purely additive from the application's perspective: a newly created physical connection whose `RegisteredApplication` is unset reports `Unknown` (`0`). On the wire the field itself is new — USERAGENT payload v1 carried no application identifier, while v2 always emits one. +- Set the property before calling `Open` or `OpenAsync`. Assigning it while the connection is connecting or open throws `InvalidOperationException`. + +```c# +using var connection = new SqlConnection(connectionString); +connection.RegisteredApplication = RegisteredApplication.EntityFrameworkCore; +await connection.OpenAsync(); +``` + +- The enum is `ushort`-backed and marked `[CLSCompliant(false)]`. Values are partitioned by range: `0x0001`–`0x7FFF` for Microsoft-defined large-scale applications, `0x8000`–`0xBFFF` for small-scale use, and `0xC000`–`0xFFFF` for public/developer use. Applications that are not yet registered can cast an unassigned value from the appropriate range. +- Application identity is **client-supplied telemetry and must never be used for authorization or any other security decision.** +- The value is not part of the connection pool key. A pooled physical connection reports the application that originally created it, and background `Min Pool Size` connections report `Unknown`. Cloned connections preserve the value. + +### Changed + +#### `TransparentNetworkIPResolution` Is Now Obsolete + +*What Changed:* + +- `SqlConnectionStringBuilder.TransparentNetworkIPResolution` is now marked `[Obsolete]`. The obsoletion message directs callers to `MultiSubnetFailover` and notes that Transparent Network IP Resolution (TNIR) is a .NET Framework-only feature. + ([#4494](https://github.com/dotnet/SqlClient/issues/4494), [#4576](https://github.com/dotnet/SqlClient/pull/4576)) + +*Who Benefits:* + +- Applications still relying on TNIR get a compile-time signal to move to `MultiSubnetFailover`, which addresses the same "connect quickly across multiple DNS-resolved addresses" goal, works consistently on every supported target framework, and is the documented strategy for Always On availability group listeners. + +*Impact:* + +- **No runtime behavior change.** TNIR still defaults to `true` on .NET Framework, and `MultiSubnetFailover` still defaults to `false`. The property remains .NET Framework-only and is not exposed on modern .NET, where a connection string containing the `Transparent Network IP Resolution` keyword still throws `NotSupportedException`. No new AppContext switches were introduced. +- The only visible effect is a new `CS0618` build warning for code that references the property. Suppress it, or migrate to `MultiSubnetFailover`, at your own pace. Flipping the TNIR and `MultiSubnetFailover` defaults is deferred to a future major version. + +#### Other changes + +- Updated `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` to v7.1.0 (was v7.1.0-preview3.26226.3). + ([#4698](https://github.com/dotnet/SqlClient/pull/4698)) + +- Unified the exception message raised when conflicting token-based and SSPI authentication properties are set on the same `SqlConnection`, and documented the complete set of properties that conflict with `AccessToken`. + ([#4629](https://github.com/dotnet/SqlClient/pull/4629)) + +- Documentation corrections: + - Rewrote the `SqlDataRecord` and `SqlMetaData` documentation, which incorrectly described the SQL CLR-only `SqlContext` and `SqlPipe` types, and clarified whether `SqlDataRecord` instances can be reused. + ([#1805](https://github.com/dotnet/SqlClient/issues/1805), [#4440](https://github.com/dotnet/SqlClient/pull/4440)) + - Corrected the LCID 1033 sample comment to use the official Windows locale name `English (United States)`. + ([#4646](https://github.com/dotnet/SqlClient/pull/4646)) + +### Fixed + +- Fixed a pooled connection being returned to the pool in a broken state after a `TransactionScope` rollback — for example, when distributed transaction promotion fails on .NET 8+ where implicit distributed transactions are disabled by default. A subsequent `Open()` succeeded but `BeginTransaction()` threw `InvalidOperationException` ("the connection has been broken"). Connection reset now preserves the transaction when the pooled connection is either a delegated transaction root **or** enlisted in a transaction, instead of only the latter. + ([#4001](https://github.com/dotnet/SqlClient/issues/4001), [#4557](https://github.com/dotnet/SqlClient/pull/4557)) + +- Fixed `GetSchema("DataTypes")` never reporting the SQL Server 2025 `json` type against Azure SQL. The row was filtered by a string comparison against a minimum server version of `17.00.000.0`, which Azure SQL can never satisfy because it always reports `12.00.xxxx`. The decision now uses the `json` support flag negotiated through the TDS `FEATUREEXTACK` token, which is accurate on both Azure SQL and on-premises SQL Server 2025+. + ([#4592](https://github.com/dotnet/SqlClient/issues/4592), [#4682](https://github.com/dotnet/SqlClient/pull/4682)) + +- Fixed a malformed UNC pipe path being composed for IPv6 literal server names over Named Pipes in managed SNI (for example `Server=np:::1`, `Server=np:[::1]`, or `Server=\\::1\pipe\sql\query`). A UNC path component may not contain a colon, and handing such a path to the OS could trigger an access violation inside LSASS on Windows, forcing a reboot. IPv6 literals are now transcribed to their `.ipv6-literal.net` form as defined by [MS-DTYP 2.2.57](https://learn.microsoft.com/openspecs/windows_protocols/ms-dtyp/62e862f4-2a51-452e-8eeb-dc4ff5ee33cc), and a colon-bearing host with no valid IPv6 interpretation now fails with the standard invalid-connection-string error. Colon-free host names, LocalDB, `localhost`, `.`, and IPv6 over TCP are unaffected. (net8.0/net9.0 only — this is the managed SNI counterpart to the native SNI fix) + ([#4523](https://github.com/dotnet/SqlClient/issues/4523), [#4558](https://github.com/dotnet/SqlClient/pull/4558)) + +- Fixed configurable retry logic installing a permanent, process-wide assembly-resolution handler that could interfere with unrelated assembly loading. The handler is now installed only while an explicitly configured custom retry provider is being resolved and constructed, and it probes `AppContext.BaseDirectory` instead of the current working directory. Place custom retry assemblies in the application base directory; dependencies loaded after provider construction must be resolvable through normal application dependency resolution or an application-supplied handler. (net8.0/net9.0 only — the .NET Framework path does not use `AssemblyLoadContext`) + ([#2214](https://github.com/dotnet/SqlClient/issues/2214), [#4547](https://github.com/dotnet/SqlClient/pull/4547)) + +- Fixed open/close throughput regressions in the opt-in connection pool V2 (`Switch.Microsoft.Data.SqlClient.UseConnectionPoolV2`). Acquiring an already-pooled connection no longer dispatches to the thread pool before attempting an inline, non-blocking acquisition, no longer allocates a timer-backed `CancellationTokenSource` before it is known that the caller will wait, and no longer allocates a `Task` on the synchronous completion path. The default pool is unaffected. + ([#4543](https://github.com/dotnet/SqlClient/pull/4543)) + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +This section summarizes all changes across the `7.1` preview cycle for users upgrading from the latest `7.0` stable release. Changes that were also serviced into `7.0.1`, `7.0.2`, or `7.0.3` are omitted — they are already present for `7.0.x` users. + +### Added + +#### `SqlBatch` Support on .NET Framework (net462 only) + +*What Changed:* + +- `SqlBatch`, `SqlBatchCommand`, and the related execution methods are now available on the .NET Framework target, so the batching API spans the full supported platform matrix. + ([#3926](https://github.com/dotnet/SqlClient/pull/3926)) + +*Who Benefits:* + +- Libraries and applications that multi-target .NET Framework and modern .NET can use a single data-access surface instead of maintaining separate batching strategies. + +*Impact:* + +- Purely additive; existing `SqlCommand` code is unchanged. + +#### `SqlConnection.GetSchemaAsync` + +*What Changed:* + +- Added asynchronous overloads of `SqlConnection.GetSchema` that mirror the existing synchronous shapes and honor a supplied `CancellationToken`. The .NET Framework schema code paths were unified with the .NET implementation in the process. + ([#3005](https://github.com/dotnet/SqlClient/pull/3005)) + +*Who Benefits:* + +- Applications that enumerate database metadata as part of a request pipeline no longer block a thread on synchronous I/O. + +*Impact:* + +- Additive; existing `GetSchema(...)` calls are unchanged. + +#### Asynchronous Key Store Provider APIs for Always Encrypted + +*What Changed:* + +- Added four `virtual` asynchronous counterparts to the synchronous methods on `SqlColumnEncryptionKeyStoreProvider`: `DecryptColumnEncryptionKeyAsync`, `EncryptColumnEncryptionKeyAsync`, `SignColumnMasterKeyMetadataAsync`, and `VerifyColumnMasterKeyMetadataAsync`. Each accepts an optional `CancellationToken`, and the default implementations delegate to the existing synchronous methods. + ([#3672](https://github.com/dotnet/SqlClient/issues/3672), [#3673](https://github.com/dotnet/SqlClient/pull/3673)) + +*Who Benefits:* + +- Authors of custom key store providers backed by network-bound stores such as HSMs or cloud key vaults can implement genuinely asynchronous key operations. + +*Impact:* + +- Purely additive; existing providers compile and run unmodified. +- **Introduced, but not yet consumed by the driver.** The driver's own column encryption key resolution path still invokes the synchronous provider methods on both the synchronous and asynchronous command paths. **A future release will enable their use from the driver's own asynchronous APIs.** +- The in-box `SqlColumnEncryptionAzureKeyVaultProvider` overrides all four methods. See the [AzureKeyVaultProvider 7.1.0](../add-ons/AzureKeyVaultProvider/7.1/7.1.0.md) release notes. ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +#### Configurable Idle Connection Timeout + +*What Changed:* + +- Added a `Connection Idle Timeout` connection-string keyword and matching `SqlConnectionStringBuilder.IdleTimeout` property that let the pool evict connections whose idle time exceeds the configured value. The default is `300` seconds; `0` disables idle expiration and negative values throw `ArgumentException`. Enforcement is gated on `Switch.Microsoft.Data.SqlClient.UseLegacyIdleTimeoutBehavior`, which defaults to `true` to preserve historical pooling behavior. + ([#4295](https://github.com/dotnet/SqlClient/pull/4295)) + +*Who Benefits:* + +- Applications running against Azure SQL Database and other elastic backends can bound connection idle time to align with server-side session recycling and reduce stale-connection failures. + +*Impact:* + +- Default behavior is unchanged: the keyword is parsed but not enforced until the legacy switch is set to `false`. + +#### Connection Pool V2 Feature Completeness + +*What Changed:* + +- Substantially expanded `ChannelDbConnectionPool`, the opt-in pool behind `Switch.Microsoft.Data.SqlClient.UseConnectionPoolV2`, bringing it to parity with the default `WaitHandleDbConnectionPool`: transaction support, broken-connection replacement, background warmup and replenishment to `Min Pool Size`, idle pruning derived from `Connection Idle Timeout`, optional connection-creation rate limiting, leaked-connection reclamation, and metrics/tracing parity. + ([#4395](https://github.com/dotnet/SqlClient/pull/4395), + [#4396](https://github.com/dotnet/SqlClient/pull/4396), + [#4429](https://github.com/dotnet/SqlClient/pull/4429), + [#4452](https://github.com/dotnet/SqlClient/pull/4452), + [#4463](https://github.com/dotnet/SqlClient/pull/4463), + [#4487](https://github.com/dotnet/SqlClient/pull/4487), + [#4504](https://github.com/dotnet/SqlClient/pull/4504), + [#4529](https://github.com/dotnet/SqlClient/pull/4529), + [#4543](https://github.com/dotnet/SqlClient/pull/4543)) +- `SqlConnection.ClearPool(SqlConnection)` and `SqlConnection.ClearAllPools()` now work correctly under pool V2. + ([#4194](https://github.com/dotnet/SqlClient/pull/4194)) + +*Who Benefits:* + +- Applications evaluating the V2 pool can exercise transaction-enlisted workloads, broken-connection recovery, leaked-connection reclamation, and pool warmup, and can observe the pool through the existing performance counters and EventSource traces. + +*Impact:* + +- No change to default behavior. Connection pool V2 remains **in evaluation** and is used only when the AppContext switch is enabled. + +#### Other additions + +- Added application identity reporting through `SqlConnection.RegisteredApplication` and USERAGENT payload version 2 (see *Changes Since 7.1.0-preview3* above). + ([#4632](https://github.com/dotnet/SqlClient/pull/4632)) + +- `SqlBulkCopy` column mappings now accept the SQL Graph pseudo-column aliases `$node_id`, `$edge_id`, `$from_id`, and `$to_id` as destination column names. + ([#3677](https://github.com/dotnet/SqlClient/pull/3677)) + +- `SqlBatchCommand.CommandBehavior` is now honored inside a `SqlBatch`, and `SqlBatch.ExecuteReader` respects the `CommandBehavior` passed to it. Batches that previously set the property and relied on it being ignored will now see it applied. + ([#4125](https://github.com/dotnet/SqlClient/pull/4125)) + +- Added connection-string synonyms for better compatibility with other SQL Server drivers: `ColumnEncryption`, `ConnectTimeout`, `FailoverPartner`, `PacketSize`, and `WorkstationId`. + ([#4192](https://github.com/dotnet/SqlClient/pull/4192)) + +- Added the SQL Server 2025 `json` data type to the `DataTypes` collection returned by `SqlConnection.GetSchema`. + ([#3858](https://github.com/dotnet/SqlClient/pull/3858)) + +### Changed + +#### Connection Timeout Can Now Propagate Through the Pool + +*What Changed:* + +- Replaced raw `TimeSpan` timeouts with a shared `TimeoutTimer` across `SqlConnection.Open[Async]`, pool acquisition, and physical connection creation, so the `Connect Timeout` budget can be deducted while a request waits in the pool. Enforcement is gated on `Switch.Microsoft.Data.SqlClient.UseOverallConnectTimeoutForPoolWait`, which defaults to `false`. Introduces a dependency on `Microsoft.Bcl.TimeProvider`. + ([#4270](https://github.com/dotnet/SqlClient/pull/4270)) + +*Who Benefits:* + +- Applications that observe pool contention can opt in to having the configured `Connect Timeout` respected end-to-end instead of the budget effectively restarting when a physical connection is eventually opened. + +*Impact:* + +- Default behavior is unchanged. When the switch is enabled, `Open`/`OpenAsync` under heavy pool contention may surface timeouts sooner than before; successful opens are unaffected. + +#### `TransparentNetworkIPResolution` Is Now Obsolete + +- See *Changes Since 7.1.0-preview3* above. + ([#4494](https://github.com/dotnet/SqlClient/issues/4494), [#4576](https://github.com/dotnet/SqlClient/pull/4576)) + +#### Other changes + +- Windows-only native SNI types are now removed cleanly by the IL trimmer on Linux and macOS, and `LocalAppContextSwitches.UseManagedNetworking` is substituted for a constant. The driver builds a single OS-agnostic assembly for all platforms; NuGet package structure and contents are unchanged. + ([#4207](https://github.com/dotnet/SqlClient/pull/4207), + [#4239](https://github.com/dotnet/SqlClient/issues/4239), + [#4465](https://github.com/dotnet/SqlClient/pull/4465), + [#4474](https://github.com/dotnet/SqlClient/pull/4474)) + +- Performance and allocation improvements: + - Restored reuse of `PacketData` linked-list nodes via a bounded free list on `StateSnapshot`, returning `SqlCommand/ExecuteReaderAsync` from +120.9% allocated against the 6.1.6 baseline to +0.1%. + ([#4536](https://github.com/dotnet/SqlClient/pull/4536)) + - `SqlBulkCopy` no longer builds SQL Graph column alias mapping tables when neither the source nor destination table contains graph pseudo-columns. + ([#4535](https://github.com/dotnet/SqlClient/pull/4535)) + - Reduced allocations when sending large string values to SQL Server, when reading `SqlErrorCollection` counters with no errors present, and on expected `null`-return paths that previously materialized stack traces. + ([#4072](https://github.com/dotnet/SqlClient/pull/4072), + [#4157](https://github.com/dotnet/SqlClient/pull/4157), + [#4099](https://github.com/dotnet/SqlClient/pull/4099), + [#4102](https://github.com/dotnet/SqlClient/pull/4102)) + - Use hardcoded LCID mappings when decoding strings, avoiding repeated culture lookups. + ([#4212](https://github.com/dotnet/SqlClient/pull/4212)) + +- Internal hardening and refactoring: + - `SqlConnection` internal state transitions now use `Interlocked.CompareExchange` guards. + ([#4267](https://github.com/dotnet/SqlClient/pull/4267)) + - Removed legacy connection-options inheritance from internal APIs and refactored `ForceNewConnection` handling. + ([#4235](https://github.com/dotnet/SqlClient/pull/4235), + [#4237](https://github.com/dotnet/SqlClient/pull/4237), + [#4261](https://github.com/dotnet/SqlClient/pull/4261), + [#4415](https://github.com/dotnet/SqlClient/pull/4415)) + - Added async generic helpers to reduce duplication across sync/async code paths. + ([#4334](https://github.com/dotnet/SqlClient/pull/4334)) + +- `SqlVector` now serializes and deserializes little-endian multibyte values explicitly for consistent behavior across architectures. + ([#3861](https://github.com/dotnet/SqlClient/pull/3861)) + +- Improved `EnclaveDiffieHellmanInfo.Size` accuracy. + ([#4346](https://github.com/dotnet/SqlClient/pull/4346)) + +- Documentation corrections for `SqlDataRecord`, `SqlMetaData`, server certificate configuration, and the LCID 1033 locale name. + ([#4408](https://github.com/dotnet/SqlClient/pull/4408), + [#4440](https://github.com/dotnet/SqlClient/pull/4440), + [#4646](https://github.com/dotnet/SqlClient/pull/4646)) + +- Updated Dependencies: + - Updated `Microsoft.Bcl.Cryptography`, `Microsoft.Extensions.Caching.Memory`, `System.Configuration.ConfigurationManager`, and `System.Security.Cryptography.Pkcs` to v9.0.18 for the `net9.0` target framework. Non-`net9.0` targets keep their existing `8.0.x` pins. + ([#4507](https://github.com/dotnet/SqlClient/pull/4507)) + - Added `System.Threading.RateLimiting` and `Microsoft.Bcl.TimeProvider` to the packaged dependency metadata. + ([#4270](https://github.com/dotnet/SqlClient/pull/4270), + [#4507](https://github.com/dotnet/SqlClient/pull/4507)) + - Updated `Microsoft.Data.SqlClient.SNI` and `Microsoft.Data.SqlClient.SNI.runtime` to v7.1.0 (was v6.0.3). + ([#4564](https://github.com/dotnet/SqlClient/pull/4564), + [#4698](https://github.com/dotnet/SqlClient/pull/4698)) + +### Fixed + +- Fixed Always Encrypted reads of `CekMdVersion` and `EkValueCount` to align with the TDS specification. + ([#4240](https://github.com/dotnet/SqlClient/pull/4240)) + +- Fixed an `OverflowException` when sending large `decimal` values (for example `decimal.MaxValue`) as a parameter with explicit `Precision` and `Scale`. This primarily affected Always Encrypted scenarios, where both must always be set. + ([#1655](https://github.com/dotnet/SqlClient/issues/1655), [#4443](https://github.com/dotnet/SqlClient/pull/4443)) + +- Fixed a TDS stream error when passing a `DateOnly` value as a parameter with `SqlDbType.Variant`, and fixed `DateOnly` values written to a `sql_variant` column of a table-valued parameter being sent as `datetime` instead of `date` (which also caused overflows for values valid for `date` but out of range for `datetime`). Reading continues to return `DateTime` instances by default for backwards compatibility. (net8.0/net9.0 only — .NET Framework has no `DateOnly` type) + ([#3953](https://github.com/dotnet/SqlClient/issues/3953), + [#3934](https://github.com/dotnet/SqlClient/issues/3934), + [#4294](https://github.com/dotnet/SqlClient/pull/4294), + [#4439](https://github.com/dotnet/SqlClient/pull/4439)) + +- Fixed a `SqlConnectionFactory` timer that woke the process every 30 seconds for the lifetime of the application even when no connection pools existed — including with `Pooling=False` and after `ClearAllPools()`. The pruning timer is now armed on demand and disarmed once there is nothing left to prune. A missing .NET Framework unload hook was also added. + ([#1881](https://github.com/dotnet/SqlClient/issues/1881), [#4479](https://github.com/dotnet/SqlClient/pull/4479)) + +- Fixed connection pool performance counter defects affecting the **default** pool as well as pool V2. `active-soft-connects` and `number-of-active-connections` could go negative after a failed connection activation, and `active-soft-connects`, `active-hard-connections`, and `number-of-pooled-connections` drifted upward permanently after a broken connection was replaced. + ([#4504](https://github.com/dotnet/SqlClient/pull/4504)) + +- Fixed a pooled connection being returned to the pool in a broken state after a `TransactionScope` rollback, which caused a later `BeginTransaction()` to throw `InvalidOperationException`. + ([#4001](https://github.com/dotnet/SqlClient/issues/4001), [#4557](https://github.com/dotnet/SqlClient/pull/4557)) + +- Fixed `GetSchema("DataTypes")` never reporting the `json` type against Azure SQL. + ([#4592](https://github.com/dotnet/SqlClient/issues/4592), [#4682](https://github.com/dotnet/SqlClient/pull/4682)) + +- Fixed a malformed UNC pipe path being composed for IPv6 literal server names over Named Pipes in managed SNI, which could trigger an LSASS access violation and a forced reboot on Windows. (net8.0/net9.0 only) + ([#4523](https://github.com/dotnet/SqlClient/issues/4523), [#4558](https://github.com/dotnet/SqlClient/pull/4558)) + +- Fixed several async entry points in `SqlBulkCopy`, `SqlDataReader.InvokeAsyncCall`, `SqlCommand.Reader`, and `SqlCommand.Xml` that captured fatal exceptions such as `OutOfMemoryException` into faulted `Task`s instead of letting them propagate. + ([#4437](https://github.com/dotnet/SqlClient/pull/4437)) + +- Fixed a `SqlDataReader` streaming bug where calling `IsDBNull()` before reading a streamed value could skip column data. + ([#4082](https://github.com/dotnet/SqlClient/pull/4082)) + +- Fixed a race in `SqlConnection.TryOpenInner` that could surface as `InvalidCastException`; the same race now returns a deterministic `InvalidOperationException`. + ([#4179](https://github.com/dotnet/SqlClient/pull/4179)) + +- Fixed `LoginWithFailover` to validate parser state before continuing, preventing null-reference failures during failover login. + ([#4140](https://github.com/dotnet/SqlClient/pull/4140)) + +- Fixed the SPN used during login to use the resolved port instead of the instance name when `Protocol=None` or `Protocol=Admin` is specified. + ([#4180](https://github.com/dotnet/SqlClient/pull/4180)) + +- Fixed several `CancellationTokenSource` leaks in `SqlDataReader`, `SqlConnection`, the `SqlCommand` reconnect paths, and the sequential-stream helpers. + ([#4009](https://github.com/dotnet/SqlClient/pull/4009)) + +### Removed + +#### SQL Server 7.0 and 2000 Support + +*What Changed:* + +- Removed dead protocol-level code paths for SQL Server 7.0 and SQL Server 2000, along with the now-orphaned `SQL Server 2000` type-system compatibility option. The `TypeSystem.SQLServer2000` enum value and the `Type System Version=SQL Server 2000` connection-string branch are gone. + ([#4015](https://github.com/dotnet/SqlClient/pull/4015)) + +*Who Benefits:* + +- The driver no longer carries dead legacy code, and the documented connection-string surface (`Latest`, `SQL Server 2005`, `SQL Server 2008`, `SQL Server 2012`) now matches the implementation exactly. + +*Impact:* + +- **Breaking:** A connection string that specifies `Type System Version=SQL Server 2000` now throws `ArgumentException` when the connection is opened. Switch to a supported value such as `Latest`. There is no change to which servers the driver connects to — SQL Server 7.0 and 2000 were already rejected during login version negotiation. + +## Contributors + +We thank the following public contributors. Their efforts toward this project are very much appreciated. + +- [edwardneal](https://github.com/edwardneal) +- [Mahdigln](https://github.com/Mahdigln) + +## Target Platform Support + +- .NET Framework 4.6.2+ (Windows x86, Windows x64, Windows ARM64) +- .NET 8.0+ (Windows x86, Windows x64, Windows ARM, Windows ARM64, Linux, macOS) + +### Dependencies + +#### .NET 9.0 + +- Microsoft.Bcl.Cryptography 9.0.18 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Data.SqlClient.SNI.runtime 7.1.0 +- Microsoft.Extensions.Caching.Memory 9.0.18 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 9.0.18 +- System.Security.Cryptography.Pkcs 9.0.18 +- System.Threading.RateLimiting 9.0.18 + +#### .NET 8.0 + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Data.SqlClient.SNI.runtime 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Threading.RateLimiting 8.0.0 + +#### .NET Standard 2.0 + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Bcl.TimeProvider 8.0.1 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Data.SqlClient.SNI.runtime 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- Microsoft.SqlServer.Server 1.0.0 +- System.Configuration.ConfigurationManager 8.0.1 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 10.0.3 +- System.Threading.Channels 10.0.3 +- System.Threading.RateLimiting 8.0.0 + +#### .NET Framework 4.6.2+ + +- Microsoft.Bcl.Cryptography 8.0.0 +- Microsoft.Bcl.TimeProvider 8.0.1 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Data.SqlClient.SNI 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.IdentityModel.JsonWebTokens 8.16.0 +- Microsoft.IdentityModel.Protocols.OpenIdConnect 8.16.0 +- System.Buffers 4.6.1 +- System.Data.Common 4.3.0 +- System.Diagnostics.DiagnosticSource 10.0.3 +- System.Memory 4.6.3 +- System.Runtime.InteropServices.RuntimeInformation 4.3.0 +- System.Security.Cryptography.Pkcs 8.0.1 +- System.Text.Json 10.0.3 +- System.Threading.Channels 10.0.3 +- System.Threading.RateLimiting 8.0.0 +- System.ValueTuple 4.6.2 diff --git a/release-notes/7.1/README.md b/release-notes/7.1/README.md index 0b7de8156d..b2c03f09f3 100644 --- a/release-notes/7.1/README.md +++ b/release-notes/7.1/README.md @@ -7,3 +7,4 @@ The following Microsoft.Data.SqlClient 7.1 releases have been shipped: | 2026-04-29 | 7.1.0-preview1 | [Release Notes](7.1.0-preview1.md) | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/release-notes/Extensions/Abstractions/7.0/7.0.3.md b/release-notes/Extensions/Abstractions/7.0/7.0.3.md new file mode 100644 index 0000000000..6676a750aa --- /dev/null +++ b/release-notes/Extensions/Abstractions/7.0/7.0.3.md @@ -0,0 +1,17 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +This release version-aligns `Microsoft.Data.SqlClient.Extensions.Abstractions` with the core [Microsoft.Data.SqlClient 7.0.3](../../../7.0/7.0.3.md) driver. The previous release of this package was [7.0.2](7.0.2.md). + +There are no functional or API changes in this release. Its `AssemblyVersion` remains `7.0.0.0`, unchanged from `7.0.2`. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 diff --git a/release-notes/Extensions/Abstractions/7.0/README.md b/release-notes/Extensions/Abstractions/7.0/README.md index 24f9eff90a..37fc312f2e 100644 --- a/release-notes/Extensions/Abstractions/7.0/README.md +++ b/release-notes/Extensions/Abstractions/7.0/README.md @@ -7,4 +7,5 @@ The following `Microsoft.Data.SqlClient.Extensions.Abstractions` | Release Date | Description | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | diff --git a/release-notes/Extensions/Abstractions/7.1/7.1.0.md b/release-notes/Extensions/Abstractions/7.1/7.1.0.md new file mode 100644 index 0000000000..f5572308c6 --- /dev/null +++ b/release-notes/Extensions/Abstractions/7.1/7.1.0.md @@ -0,0 +1,27 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +This release continues version-alignment of `Microsoft.Data.SqlClient.Extensions.Abstractions` with the core `Microsoft.Data.SqlClient` driver version (`7.1.0`). The previous release of this package was [7.1.0-preview3](7.1.0-preview3.md), and the previous stable release was [7.0.3](../7.0/7.0.3.md). + +There are no functional or API changes in this release. See the core [Microsoft.Data.SqlClient 7.1.0](../../../7.1/7.1.0.md) release notes for the driver-family changes shipped alongside this version. + +> **Version alignment:** This package's version continues to track the core `Microsoft.Data.SqlClient` driver version. See the [7.0.2 release notes](../../../7.0/7.0.2.md) for the initial alignment announcement. This assembly ships with `FileVersion 7.1.0.x`. The `AssemblyVersion 7.0.0.0` remains unchanged from the `7.0.2` release, so upgrading from `7.0.2`, `7.0.3`, or any `7.1` preview does not require new .NET Framework strong-name binding redirects. This package's `AssemblyVersion` was raised from `1.0.0.0` to `7.0.0.0` in `7.0.2`, so applications moving from the earlier `1.0.0` package may still need the one-time .NET Framework binding redirect described in those notes. + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +There are no functional or API changes in this release. + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +There are no functional or API changes in this package across the `7.1` preview cycle. Every `7.1` release of this package has been a version-alignment release. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 diff --git a/release-notes/Extensions/Abstractions/7.1/README.md b/release-notes/Extensions/Abstractions/7.1/README.md index 9d1374bdb6..20ba3ecff0 100644 --- a/release-notes/Extensions/Abstractions/7.1/README.md +++ b/release-notes/Extensions/Abstractions/7.1/README.md @@ -7,3 +7,4 @@ The following `Microsoft.Data.SqlClient.Extensions.Abstractions` | :-- | :-- | :--: | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/release-notes/Extensions/Azure/7.0/7.0.3.md b/release-notes/Extensions/Azure/7.0/7.0.3.md new file mode 100644 index 0000000000..436930ac89 --- /dev/null +++ b/release-notes/Extensions/Azure/7.0/7.0.3.md @@ -0,0 +1,39 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +This update brings the following changes since the [7.0.2](7.0.2.md) release of `Microsoft.Data.SqlClient.Extensions.Azure`. + +This package ships version-aligned with the core [Microsoft.Data.SqlClient 7.0.3](../../../7.0/7.0.3.md) driver. Its `AssemblyVersion` remains `7.0.0.0`, unchanged from `7.0.2`. + +### Fixed + +- Fixed Entra ID authentication failures when the server supplies an authority URL with an OAuth endpoint suffix, such as `https://login.microsoftonline.com//oauth2/authorize`. The provider now extracts the tenant from the first path segment and normalizes the authority used by Azure.Identity and MSAL, restoring authentication to Dataverse/Dynamics 365 TDS endpoints, including `Active Directory Service Principal` authentication. Malformed authority URLs now produce a clear authentication error. Bare tenant authority URLs used by Azure SQL and Fabric retain their existing behavior. + ([#4496](https://github.com/dotnet/SqlClient/issues/4496), [#4521](https://github.com/dotnet/SqlClient/pull/4521), [#4572](https://github.com/dotnet/SqlClient/pull/4572), [#4641](https://github.com/dotnet/SqlClient/pull/4641)) + +## Target Platform Support + +- .NET Standard 2.0 +- .NET Framework 4.6.2+ + +### Dependencies + +#### .NET Standard 2.0 + +- Azure.Core 1.51.1 +- Azure.Identity 1.18.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 + +#### .NET Framework 4.6.2+ + +- Azure.Core 1.51.1 +- Azure.Identity 1.18.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 diff --git a/release-notes/Extensions/Azure/7.0/README.md b/release-notes/Extensions/Azure/7.0/README.md index 93b90c690d..b462b02a57 100644 --- a/release-notes/Extensions/Azure/7.0/README.md +++ b/release-notes/Extensions/Azure/7.0/README.md @@ -7,4 +7,5 @@ The following `Microsoft.Data.SqlClient.Extensions.Azure` | Release Date | Description | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | diff --git a/release-notes/Extensions/Azure/7.1/7.1.0.md b/release-notes/Extensions/Azure/7.1/7.1.0.md new file mode 100644 index 0000000000..3c73bb6631 --- /dev/null +++ b/release-notes/Extensions/Azure/7.1/7.1.0.md @@ -0,0 +1,45 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +This release continues version-alignment of `Microsoft.Data.SqlClient.Extensions.Azure` with the core `Microsoft.Data.SqlClient` driver version (`7.1.0`). The previous release of this package was [7.1.0-preview3](7.1.0-preview3.md), and the previous stable release was [7.0.3](../7.0/7.0.3.md). See the core [Microsoft.Data.SqlClient 7.1.0](../../../7.1/7.1.0.md) release notes for the driver-family changes shipped alongside this version. + +> **Version alignment:** This package's version continues to track the core `Microsoft.Data.SqlClient` driver version. See the [7.0.2 release notes](../../../7.0/7.0.2.md) for the initial alignment announcement. This assembly ships with `FileVersion 7.1.0.x`. The `AssemblyVersion 7.0.0.0` remains unchanged from the `7.0.2` release, so upgrading from `7.0.2`, `7.0.3`, or any `7.1` preview does not require new .NET Framework strong-name binding redirects. This package's `AssemblyVersion` was raised from `1.0.0.0` to `7.0.0.0` in `7.0.2`, so applications moving from the earlier `1.0.0` package may still need the one-time .NET Framework binding redirect described in those notes. + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +### Changed + +- Clarified the internal Entra ID authority handling in `ActiveDirectoryAuthenticationProvider` by keeping the server-supplied STSURL, the Azure Identity authority URL, and the normalized MSAL authority distinct, and by composing the MSAL authority only on the MSAL-based code paths. There is no behavior change and no public API change. + ([#4630](https://github.com/dotnet/SqlClient/pull/4630)) + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +There are no additional cumulative changes since [7.0.3](../7.0/7.0.3.md). + +## Target Platform Support + +- .NET Standard 2.0 +- .NET Framework 4.6.2+ + +### Dependencies + +#### .NET Standard 2.0 + +- Azure.Core 1.51.1 +- Azure.Identity 1.18.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 + +#### .NET Framework 4.6.2+ + +- Azure.Core 1.51.1 +- Azure.Identity 1.18.0 +- Microsoft.Data.SqlClient.Extensions.Abstractions 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 +- Microsoft.Identity.Client 4.84.2 +- Microsoft.Identity.Client.Broker 4.84.2 diff --git a/release-notes/Extensions/Azure/7.1/README.md b/release-notes/Extensions/Azure/7.1/README.md index b3f93b7931..7cb63309db 100644 --- a/release-notes/Extensions/Azure/7.1/README.md +++ b/release-notes/Extensions/Azure/7.1/README.md @@ -7,3 +7,4 @@ The following `Microsoft.Data.SqlClient.Extensions.Azure` | :-- | :-- | :--: | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/release-notes/Internal/Logging/7.0/7.0.3.md b/release-notes/Internal/Logging/7.0/7.0.3.md new file mode 100644 index 0000000000..57dfbd1610 --- /dev/null +++ b/release-notes/Internal/Logging/7.0/7.0.3.md @@ -0,0 +1,20 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +> **Note:** This package is for internal use by other Microsoft.Data.SqlClient packages only +> and should not be referenced directly by application code. + +This release version-aligns `Microsoft.Data.SqlClient.Internal.Logging` with the core [Microsoft.Data.SqlClient 7.0.3](../../../7.0/7.0.3.md) driver. The previous release of this package was [7.0.2](7.0.2.md). + +There are no functional or API changes in this release. Its `AssemblyVersion` remains `7.0.0.0`, unchanged from `7.0.2`. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- None diff --git a/release-notes/Internal/Logging/7.0/README.md b/release-notes/Internal/Logging/7.0/README.md index 403c61db07..c4a5c3253b 100644 --- a/release-notes/Internal/Logging/7.0/README.md +++ b/release-notes/Internal/Logging/7.0/README.md @@ -10,4 +10,5 @@ The following `Microsoft.Data.SqlClient.Internal.Logging` | Release Date | Description | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | diff --git a/release-notes/Internal/Logging/7.1/7.1.0.md b/release-notes/Internal/Logging/7.1/7.1.0.md new file mode 100644 index 0000000000..797e5b9050 --- /dev/null +++ b/release-notes/Internal/Logging/7.1/7.1.0.md @@ -0,0 +1,30 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +> **Note:** This package is for internal use by other Microsoft.Data.SqlClient packages only +> and should not be referenced directly by application code. + +This release continues version-alignment of `Microsoft.Data.SqlClient.Internal.Logging` with the core `Microsoft.Data.SqlClient` driver version (`7.1.0`). The previous release of this package was [7.1.0-preview3](7.1.0-preview3.md), and the previous stable release was [7.0.3](../7.0/7.0.3.md). + +There are no functional or API changes in this release. See the core [Microsoft.Data.SqlClient 7.1.0](../../../7.1/7.1.0.md) release notes for the driver-family changes shipped alongside this version. + +> **Version alignment:** This package's version continues to track the core `Microsoft.Data.SqlClient` driver version. See the [7.0.2 release notes](../../../7.0/7.0.2.md) for the initial alignment announcement. This assembly ships with `FileVersion 7.1.0.x`. The `AssemblyVersion 7.0.0.0` remains unchanged from the `7.0.2` release, so upgrading from `7.0.2`, `7.0.3`, or any `7.1` preview does not require new .NET Framework strong-name binding redirects. This package's `AssemblyVersion` was raised from `1.0.0.0` to `7.0.0.0` in `7.0.2`, so applications moving from the earlier `1.0.0` package may still need the one-time .NET Framework binding redirect described in those notes. + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +There are no functional or API changes in this release. + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +There are no functional or API changes in this package across the `7.1` preview cycle. Every `7.1` release of this package has been a version-alignment release. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- None diff --git a/release-notes/Internal/Logging/7.1/README.md b/release-notes/Internal/Logging/7.1/README.md index 25caeccaba..005fe0e118 100644 --- a/release-notes/Internal/Logging/7.1/README.md +++ b/release-notes/Internal/Logging/7.1/README.md @@ -10,3 +10,4 @@ The following `Microsoft.Data.SqlClient.Internal.Logging` | :-- | :-- | :--: | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/release-notes/README.md b/release-notes/README.md index c32e0bc2be..600cad636f 100644 --- a/release-notes/README.md +++ b/release-notes/README.md @@ -1,6 +1,6 @@ # Microsoft.Data.SqlClient Release Notes -The latest stable release is [Microsoft.Data.SqlClient 7.0](7.0). +The latest stable release is [Microsoft.Data.SqlClient 7.1](7.1). ## Release Information @@ -23,7 +23,7 @@ The latest stable release is [Microsoft.Data.SqlClient 7.0](7.0). # Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider Release Notes The latest stable release is -[Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.0](add-ons/AzureKeyVaultProvider/7.0). +[Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider 7.1](add-ons/AzureKeyVaultProvider/7.1). ## Release Information @@ -41,7 +41,7 @@ The latest stable release is # Microsoft.Data.SqlClient.Extensions.Abstractions Release Notes The latest release is -[Microsoft.Data.SqlClient.Extensions.Abstractions 7.0](Extensions/Abstractions/7.0). +[Microsoft.Data.SqlClient.Extensions.Abstractions 7.1](Extensions/Abstractions/7.1). ## Release Information @@ -52,7 +52,7 @@ The latest release is # Microsoft.Data.SqlClient.Extensions.Azure Release Notes The latest release is -[Microsoft.Data.SqlClient.Extensions.Azure 7.0](Extensions/Azure/7.0). +[Microsoft.Data.SqlClient.Extensions.Azure 7.1](Extensions/Azure/7.1). ## Release Information @@ -64,7 +64,7 @@ The latest release is # Microsoft.Data.SqlClient.Internal.Logging Release Notes The latest release is -[Microsoft.Data.SqlClient.Internal.Logging 7.0](Internal/Logging/7.0). +[Microsoft.Data.SqlClient.Internal.Logging 7.1](Internal/Logging/7.1). ## Release Information diff --git a/release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md b/release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md new file mode 100644 index 0000000000..1864b0ce42 --- /dev/null +++ b/release-notes/add-ons/AzureKeyVaultProvider/7.0/7.0.3.md @@ -0,0 +1,21 @@ +# Release Notes + +## Stable Release 7.0.3 - 2026-09-10 + +This release version-aligns `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` with the core [Microsoft.Data.SqlClient 7.0.3](../../../7.0/7.0.3.md) driver. The previous release of this package was [7.0.2](7.0.2.md). + +There are no functional or API changes in this release. Its `AssemblyVersion` remains `7.0.0.0`, unchanged from `7.0.2`. + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- Azure.Core 1.51.1 +- Azure.Security.KeyVault.Keys 4.9.0 +- Microsoft.Data.SqlClient 7.0.3 +- Microsoft.Data.SqlClient.Internal.Logging 7.0.3 +- Microsoft.Extensions.Caching.Memory 8.0.1 diff --git a/release-notes/add-ons/AzureKeyVaultProvider/7.0/README.md b/release-notes/add-ons/AzureKeyVaultProvider/7.0/README.md index a51fe94635..febedc433b 100644 --- a/release-notes/add-ons/AzureKeyVaultProvider/7.0/README.md +++ b/release-notes/add-ons/AzureKeyVaultProvider/7.0/README.md @@ -5,6 +5,7 @@ The following `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` | Release Date | Description | Notes | | :-- | :-- | :--: | +| 2026-09-10 | 7.0.3 | [Release Notes](7.0.3.md) | | 2026-06-24 | 7.0.2 | [Release Notes](7.0.2.md) | | 2026-03-17 | 7.0.0 | [Release Notes](7.0.0.md) | | 2026-03-05 | 7.0.0-preview1.26064.3 | [Release Notes](7.0.0-preview1.md) | diff --git a/release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md b/release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md new file mode 100644 index 0000000000..67d99ba296 --- /dev/null +++ b/release-notes/add-ons/AzureKeyVaultProvider/7.1/7.1.0.md @@ -0,0 +1,58 @@ +# Release Notes + +## Stable Release 7.1.0 - 2026-09-17 + +This release continues version-alignment of `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` with the core `Microsoft.Data.SqlClient` driver version (`7.1.0`). The previous release of this package was [7.1.0-preview3](7.1.0-preview3.md), and the previous stable release was [7.0.3](../7.0/7.0.3.md). + +There are no functional or API changes since [7.1.0-preview3](7.1.0-preview3.md). See the core [Microsoft.Data.SqlClient 7.1.0](../../../7.1/7.1.0.md) release notes for the driver-family changes shipped alongside this version. + +> **Version alignment:** This package's version continues to track the core `Microsoft.Data.SqlClient` driver version. See the [7.0.2 release notes](../../../7.0/7.0.2.md) for the initial alignment announcement. This assembly ships with `FileVersion 7.1.0.x`. The `AssemblyVersion 7.0.0.0` remains unchanged from the `7.0.2` release, so upgrading from any `7.0.x` release does not require new .NET Framework strong-name binding redirects. + +## Changes Since [7.1.0-preview3](7.1.0-preview3.md) + +There are no functional or API changes in this release. + +## Cumulative Changes Since [7.0.3](../7.0/7.0.3.md) + +### Added + +#### Asynchronous key store provider APIs + +*What Changed:* + +- `SqlColumnEncryptionAzureKeyVaultProvider` now overrides the four asynchronous key store provider methods introduced on the base class in [#3673](https://github.com/dotnet/SqlClient/pull/3673): `EncryptColumnEncryptionKeyAsync`, `DecryptColumnEncryptionKeyAsync`, `SignColumnMasterKeyMetadataAsync`, and `VerifyColumnMasterKeyMetadataAsync`. These call the Azure SDK's own asynchronous methods and flow the supplied `CancellationToken` to them, rather than completing synchronous work on a returned task. +- Concurrent cache misses for the same key are gated so a burst of callers issues a single Key Vault request. The gate is only ever awaited, so no thread blocks, and misses for different keys still proceed in parallel. The gate applies only while caching is enabled — setting `ColumnEncryptionKeyCacheTtl` to zero disables caching and bypasses the gate, so concurrent operations for the same key may issue parallel Key Vault requests. + ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +*Who Benefits:* + +- Applications using Always Encrypted with Azure Key Vault can await column encryption key operations rather than blocking a thread pool thread on network-bound key vault I/O, when they call the provider directly. + +*Impact:* + +- No public API was removed or changed. The column encryption key and signature caches are shared between the synchronous and asynchronous paths, so a key resolved by one is visible to the other. +- **Introduced, but not yet consumed by the driver.** `Microsoft.Data.SqlClient`'s own key resolution path still invokes the synchronous provider methods on both synchronous and asynchronous command execution, so these overrides do not by themselves make Always Encrypted query execution non-blocking. **A future release will enable their use from the driver's own asynchronous APIs.** See the core [7.1.0](../../../7.1/7.1.0.md) release notes. +- **Behavior change:** `VerifyColumnMasterKeyMetadata` and `VerifyColumnMasterKeyMetadataAsync` now reject a null or empty `signature` with `ArgumentNullException` / `ArgumentException` instead of passing it through to the Azure SDK. In-product callers are unaffected because `SqlSecurityUtility.VerifyColumnMasterKeySignature` already rejects those values upstream. +- **Runtime requirement:** this version requires `Microsoft.Data.SqlClient` `7.1.0-preview3` or later at runtime, because the asynchronous base-class methods it overrides first shipped in that release. Pairing it with the aligned stable `7.1.0` is recommended, and the NuGet dependency floor resolves there. Because assembly versions unify at `major.0.0.0`, the provider also binds against a `7.1` preview driver, but running it against a driver older than `7.1.0-preview3` produces a `TypeLoadException`. + +### Fixed + +- Fixed the 2000-entry column master key signature cache being able to stop compacting and grow unbounded. `LocalCache.GetOrCreate` compacted only when `Count == maxSize`, and under concurrency that equality test could be stepped past, permanently disabling compaction. It now compacts on `Count >= maxSize`. + ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +- Fixed concurrent callers ending up with different `CryptographyClient` instances for the same key, which produced redundant clients and duplicated Key Vault handshakes. `GetCryptographyClient` now uses `GetOrAdd` instead of `TryGetValue` followed by `TryAdd`. + ([#4540](https://github.com/dotnet/SqlClient/pull/4540)) + +## Target Platform Support + +- .NET Standard 2.0 + +### Dependencies + +#### .NET Standard 2.0 + +- Azure.Core 1.51.1 +- Azure.Security.KeyVault.Keys 4.9.0 +- Microsoft.Data.SqlClient 7.1.0 +- Microsoft.Data.SqlClient.Internal.Logging 7.1.0 +- Microsoft.Extensions.Caching.Memory 8.0.1 diff --git a/release-notes/add-ons/AzureKeyVaultProvider/7.1/README.md b/release-notes/add-ons/AzureKeyVaultProvider/7.1/README.md index 07fffc069b..6232931b72 100644 --- a/release-notes/add-ons/AzureKeyVaultProvider/7.1/README.md +++ b/release-notes/add-ons/AzureKeyVaultProvider/7.1/README.md @@ -7,3 +7,4 @@ The following `Microsoft.Data.SqlClient.AlwaysEncrypted.AzureKeyVaultProvider` | :-- | :-- | :--: | | 2026-07-09 | 7.1.0-preview2 | [Release Notes](7.1.0-preview2.md) | | 2026-08-26 | 7.1.0-preview3 | [Release Notes](7.1.0-preview3.md) | +| 2026-09-17 | 7.1.0 | [Release Notes](7.1.0.md) | diff --git a/roadmap.md b/roadmap.md deleted file mode 100644 index e41d19dc79..0000000000 --- a/roadmap.md +++ /dev/null @@ -1,92 +0,0 @@ -# Microsoft.Data.SqlClient Roadmap - -The Microsoft.Data.SqlClient roadmap communicates project priorities for evolving and extending the scope of the product. We encourage the community to work with us to improve the SqlClient driver for these scenarios and extend it for others. - -> **Last updated:** May 2026 -> -> This roadmap is a living document. Priorities and timelines may shift based on community feedback, engineering constraints, and business needs. We update this page regularly to reflect the current state of development. - ---- - -## Release Milestones - -For active release milestones, their target dates, and the changes included, see [SqlClient milestones](https://github.com/dotnet/SqlClient/milestones). - ---- - -## Current Focus Areas - -Our team is actively working on the following high-level themes. Features are tracked via [GitHub issues](https://github.com/dotnet/SqlClient/issues) where applicable — see the linked milestones for associated issue details. - -- **Active** — Currently in development -- **Planned** — Committed for a future milestone with estimated delivery -- **Backlog** — On our radar for future months, not yet scheduled - -### Performance & Reliability - -| Work Item(s) | Feature | Status | ETA | -| ------------ | ------- | ------ | --- | -| [#3356](https://github.com/dotnet/SqlClient/issues/3356) | Connection pool performance improvements | Active | July 2026 | -| N/A | Performance benchmarking suite | Active | July 2026 | -| [#422](https://github.com/dotnet/SqlClient/issues/422) | Phase 1 - Unix async performance — thread starvation in parallel `ExecuteReaderAsync` | Active | September 2026 | -| TBD | Phase 2 - Async usage analysis and optimization | Planned | — | - -### New Data Type Support - -| Work Item(s) | Feature | Status | ETA | -| ------------ | ------- | ------ | --- | -| TBD | Vector subtype support — `float16` (`Half`) | Active | August 2026 | - -### Observability & Diagnostics - -| Work Item(s) | Feature | Status | ETA | -| ------------ | ------- | ------ | --- | -| [#2210](https://github.com/dotnet/SqlClient/issues/2210) [#2211](https://github.com/dotnet/SqlClient/issues/2211) | OpenTelemetry support | Planned | — | -| N/A | Logging improvements | Planned | — | -| TBD | Integrate with / expose MSAL logging | Planned | — | - -### API Improvements - -| Work Item(s) | Feature | Status | ETA | -| ------------ | ------- | ------ | --- | -| [#2353](https://github.com/dotnet/SqlClient/issues/2353) | Expose connection encryption information to clients | Planned | September 2026 | -| [#26](https://github.com/dotnet/SqlClient/issues/26) | Throw `TaskCanceledException` instead of `SqlException` for cancellations | Planned | September 2026 | -| [#113](https://github.com/dotnet/SqlClient/issues/113) | `BeginTransactionAsync` API on `SqlConnection` | Planned | — | - -### Security & Architecture - -| Feature | Status | ETA | -| ------- | ------ | --- | -| Security hardening activities | Active | Ongoing internally | - -### AI & Developer Tooling - -| Feature | Status | ETA | -| ------- | ------ | --- | -| `System.Data.SqlClient` → `Microsoft.Data.SqlClient` migration via Modernize with Copilot | Active | September 2026 | -| Modernize SqlClient repository with AI | Active | Ongoing | - -### Engineering & Infrastructure - -| Feature | Status | ETA | -| ------- | ------ | --- | -| CI/CD pipeline redesign | Active | August 2026 | -| Add SQL Server 2025 to test matrix | Planned | August 2026 | -| Add .NET 10 to test matrix | Planned | August 2026 | -| Converting existing traditional pipelines to YAML | Active | August 2026 | -| Performance benchmarking pipeline (Internal) | Planned | September 2026 | - ---- - -## Released Versions - -- [Release Notes](release-notes/README.md) — Detailed release notes summarizing all changes and features released. -- [GitHub Releases](https://github.com/dotnet/SqlClient/releases) — NuGet packages and changelog notes for each release. - ---- - -## Community Contributions & Feedback - -For information on how to contribute, see [CONTRIBUTING.md](CONTRIBUTING.md). For details on the PR tracking workflow, see [contributing-workflow.md](contributing-workflow.md). - -The best way to give feedback is to create issues in the [dotnet/SqlClient](https://github.com/dotnet/SqlClient) repo. diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 2c4b54b5d0..d9b9a7cdc8 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -32,6 +32,71 @@ + + + + true + true + true + + latest-recommended + + false + + $(MSBuildProjectFullPath).$([System.Guid]::NewGuid().ToString()).sarif + + false + + + + + + + + + + + + + + + diff --git a/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs b/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs index 3bdce5c751..cbb0371bec 100644 --- a/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs +++ b/src/Microsoft.Data.SqlClient.Extensions/Azure/src/ActiveDirectoryAuthenticationProvider.cs @@ -252,7 +252,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti // ("https://login.microsoftonline.com/{tenantId}/oauth2/authorize"), so the tenant is // taken from the first path segment rather than the last. - if (!TryParseAuthority(parameters.Authority, out string authorityHost, out string tenant, out string msalAuthority)) + if (!TryParseAuthority(parameters.Authority, out string authorityUrl, out string tenant)) { throw new Extensions.Azure.AuthenticationException( parameters.AuthenticationMethod, @@ -266,7 +266,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryDefault) { // Cache DefaultAzureCredential based on scope, authority host, tenant, and clientId - TokenCredentialKey tokenCredentialKey = new(typeof(DefaultAzureCredential), authorityHost, scope, tenant, clientId); + TokenCredentialKey tokenCredentialKey = new(typeof(DefaultAzureCredential), authorityUrl, scope, tenant, clientId); AccessToken accessToken = await GetTokenAsync(tokenCredentialKey, string.Empty, tokenRequestContext, cts.Token).ConfigureAwait(false); SqlClientEventSource.Log.TryTraceEvent("AcquireTokenAsync | Acquired access token for Default auth mode. Expiry Time: {0}", accessToken.ExpiresOn); return new SqlAuthenticationToken(accessToken.Token, accessToken.ExpiresOn); @@ -275,7 +275,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryManagedIdentity || parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryMSI) { // Cache ManagedIdentityCredential based on scope, authority host, and clientId - TokenCredentialKey tokenCredentialKey = new(typeof(ManagedIdentityCredential), authorityHost, scope, string.Empty, clientId); + TokenCredentialKey tokenCredentialKey = new(typeof(ManagedIdentityCredential), authorityUrl, scope, string.Empty, clientId); AccessToken accessToken = await GetTokenAsync(tokenCredentialKey, string.Empty, tokenRequestContext, cts.Token).ConfigureAwait(false); SqlClientEventSource.Log.TryTraceEvent("AcquireTokenAsync | Acquired access token for Managed Identity auth mode. Expiry Time: {0}", accessToken.ExpiresOn); return new SqlAuthenticationToken(accessToken.Token, accessToken.ExpiresOn); @@ -284,7 +284,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryServicePrincipal) { // Cache ClientSecretCredential based on scope, authority host, tenant, and clientId - TokenCredentialKey tokenCredentialKey = new(typeof(ClientSecretCredential), authorityHost, scope, tenant, clientId); + TokenCredentialKey tokenCredentialKey = new(typeof(ClientSecretCredential), authorityUrl, scope, tenant, clientId); string password = parameters.Password is null ? string.Empty : parameters.Password; AccessToken accessToken = await GetTokenAsync(tokenCredentialKey, password, tokenRequestContext, cts.Token).ConfigureAwait(false); SqlClientEventSource.Log.TryTraceEvent("AcquireTokenAsync | Acquired access token for Active Directory Service Principal auth mode. Expiry Time: {0}", accessToken.ExpiresOn); @@ -294,7 +294,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryWorkloadIdentity) { // Cache WorkloadIdentityCredential based on authority host and clientId - TokenCredentialKey tokenCredentialKey = new(typeof(WorkloadIdentityCredential), authorityHost, string.Empty, string.Empty, clientId); + TokenCredentialKey tokenCredentialKey = new(typeof(WorkloadIdentityCredential), authorityUrl, string.Empty, string.Empty, clientId); // If either tenant id, client id, or the token file path are not specified when fetching the token, // a CredentialUnavailableException will be thrown instead AccessToken accessToken = await GetTokenAsync(tokenCredentialKey, string.Empty, tokenRequestContext, cts.Token).ConfigureAwait(false); @@ -322,11 +322,12 @@ public override async Task AcquireTokenAsync(SqlAuthenti #endif : s_systemBrowserRedirectUri; + string msalAuthorityUrl = authorityUrl + tenant; PublicClientAppKey pcaKey = #if NETFRAMEWORK - new(msalAuthority, redirectUri, _applicationClientId, _iWin32WindowFunc); + new(msalAuthorityUrl, redirectUri, _applicationClientId, _iWin32WindowFunc); #else - new(msalAuthority, redirectUri, _applicationClientId); + new(msalAuthorityUrl, redirectUri, _applicationClientId); #endif AuthenticationResult? result = null; @@ -364,7 +365,7 @@ public override async Task AcquireTokenAsync(SqlAuthenti else if (parameters.AuthenticationMethod == SqlAuthenticationMethod.ActiveDirectoryPassword) #pragma warning restore CS0618 // Type or member is obsolete { - string pwCacheKey = GetAccountPwCacheKey(msalAuthority, parameters.UserId); + string pwCacheKey = GetAccountPwCacheKey(msalAuthorityUrl, parameters.UserId); object? previousPw = s_accountPwCache.Get(pwCacheKey); string password = parameters.Password is null ? string.Empty : parameters.Password; byte[] currPwHash = GetHash(password); @@ -547,26 +548,23 @@ or AuthenticationRequiredException } /// - /// Splits an Entra ID authority URL (the STSURL provided by the server in the FEDAUTHINFO TDS - /// token) into the authority host and the tenant. + /// Splits the STSURL provided by the server in the FEDAUTHINFO TDS token into the Entra ID + /// authority URL and tenant. /// - /// - /// The authority URL, e.g. https://login.microsoftonline.com/{tenantId}. Some services + /// + /// The STSURL, e.g. https://login.microsoftonline.com/{tenantId}. Some services /// (for example the Dataverse/Dynamics 365 TDS endpoint) return an ADAL v1 style URL such as /// https://login.microsoftonline.com/{tenantId}/oauth2/authorize. /// - /// - /// Receives the authority host with a trailing slash, e.g. https://login.microsoftonline.com/. + /// + /// Receives the authority URL with a trailing slash, e.g. https://login.microsoftonline.com/. /// /// /// Receives the tenant (the first path segment of the authority URL), which may be a tenant id, /// a domain name, or one of the common/organizations/consumers placeholders. /// - /// - /// Receives the normalized authority (host + tenant) suitable for MSAL's WithAuthority. - /// /// - /// true if the authority URL is a well-formed, absolute HTTPS URL carrying a tenant + /// true if the STSURL is a well-formed, absolute HTTPS URL carrying a tenant /// segment; otherwise false. /// /// @@ -577,17 +575,16 @@ or AuthenticationRequiredException /// /// Entra ID authorities are always absolute HTTPS URLs, so anything else is rejected rather /// than guessed at. Both MSAL (WithAuthority) and Azure.Identity - /// (TokenCredentialOptions.AuthorityHost) require an absolute URI as well, so an + /// (TokenCredentialOptions.AuthorityHost) require an absolute URL as well, so an /// unparseable authority cannot produce a working credential. /// /// internal static bool TryParseAuthority( - string authorityUrl, - out string authorityHost, - out string tenant, - out string msalAuthority) + string stsUrl, + out string authorityUrl, + out string tenant) { - if (Uri.TryCreate(authorityUrl, UriKind.Absolute, out Uri? uri) && + if (Uri.TryCreate(stsUrl, UriKind.Absolute, out Uri? uri) && uri.Scheme == Uri.UriSchemeHttps && uri.Segments.Length > 1) { @@ -598,15 +595,13 @@ internal static bool TryParseAuthority( if (tenant.Length > 0) { - authorityHost = uri.GetLeftPart(UriPartial.Authority) + "/"; - msalAuthority = authorityHost + tenant; + authorityUrl = uri.GetLeftPart(UriPartial.Authority) + "/"; return true; } } - authorityHost = string.Empty; + authorityUrl = string.Empty; tenant = string.Empty; - msalAuthority = string.Empty; return false; } @@ -840,15 +835,15 @@ private static async Task GetTokenAsync(TokenCredentialKey tokenCre /// /// Builds the cache key used to remember which password was last validated for an account. /// - /// - /// The normalized authority (host + tenant) from . The + /// + /// The normalized MSAL authority URL (authority URL + tenant). The /// normalized form is used so that two spellings of the same tenant (for example a bare /// tenant endpoint and an OAuth v1 /oauth2/authorize endpoint) share a single entry. /// /// The user id being authenticated, which may be null. - private static string GetAccountPwCacheKey(string msalAuthority, string? userId) + private static string GetAccountPwCacheKey(string msalAuthorityUrl, string? userId) { - return msalAuthority + "+" + userId; + return msalAuthorityUrl + "+" + userId; } private static byte[] GetHash(string input) diff --git a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AADConnectionTest.cs b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AADConnectionTest.cs index a8060354d4..063db64d84 100644 --- a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AADConnectionTest.cs +++ b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AADConnectionTest.cs @@ -85,22 +85,11 @@ public static void ActiveDirectoryDefaultMustPass() ConnectAndDisconnect(connStr); } - // This test works on main in the existing jobs (like Win22_Sql22), but - // fails in the Azure project tests on a similar agent/image: - // - // Failed Microsoft.Data.SqlClient.Extensions.Azure.Test.AADConnectionTest.ADIntegratedUsingSSPI [59 ms] - // Error Message: - // Microsoft.Data.SqlClient.SqlException : Failed to authenticate the user NT Authority\Anonymous Logon in Active Directory (Authentication=ActiveDirectoryIntegrated). - // Error code 0xget_user_name_failed - // Failed to acquire access token for ActiveDirectoryIntegrated: Failed to get user name. - // - // ActiveIssue tests can be filtered out of test runs on the dotnet CLI - // using the filter "category != failing". - // - [ActiveIssue("https://sqlclientdrivers.visualstudio.com/ADO.Net/_workitems/edit/45941")] + // This test requires a Windows identity configured for Entra Integrated authentication + // to the target Azure SQL database, which is distinct from SQL Server integrated security. [ConditionalFact( typeof(Config), - nameof(Config.SupportsIntegratedSecurity), + nameof(Config.SupportsEntraIntegrated), nameof(Config.HasTcpConnectionString))] public static void ADIntegratedUsingSSPI() { diff --git a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/ActiveDirectoryInteractiveTests.cs b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/ActiveDirectoryInteractiveTests.cs index 9240d83b40..9f858a4d2b 100644 --- a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/ActiveDirectoryInteractiveTests.cs +++ b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/ActiveDirectoryInteractiveTests.cs @@ -16,7 +16,7 @@ public ActiveDirectoryInteractiveTests(ITestOutputHelper output) } [Fact] - [Trait("Category", "Interactive")] + [Trait("category", "interactive")] public async Task TestConnection() { string connectionString = new SqlConnectionStringBuilder() diff --git a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AuthorityParsingTests.cs b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AuthorityParsingTests.cs index cbbd56e3d1..bd97f3926d 100644 --- a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AuthorityParsingTests.cs +++ b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/AuthorityParsingTests.cs @@ -6,7 +6,7 @@ namespace Microsoft.Data.SqlClient.Extensions.Azure.Test; /// /// Tests for splitting the STSURL supplied by the server in the FEDAUTHINFO TDS token into an -/// authority host and a tenant. +/// authority URL and a tenant. /// /// /// The cases below only cover authority shapes that Entra ID actually documents: @@ -16,97 +16,93 @@ public class AuthorityParsingTests { private const string Tenant = "72f988bf-86f1-41af-91ab-2d7cd011db47"; - public static TheoryData AuthorityData => new() + /// + /// Provides documented STSURL shapes and their expected authority URL and tenant. + /// + public static TheoryData AuthorityData => new() { // Azure SQL / Fabric style authority. { $"https://login.microsoftonline.com/{Tenant}", "https://login.microsoftonline.com/", - Tenant, - $"https://login.microsoftonline.com/{Tenant}" + Tenant }, // Trailing slash. { $"https://login.microsoftonline.com/{Tenant}/", "https://login.microsoftonline.com/", - Tenant, - $"https://login.microsoftonline.com/{Tenant}" + Tenant }, // v1.0 authorize endpoint, as returned by the Dataverse / Dynamics 365 TDS endpoint. { $"https://login.microsoftonline.com/{Tenant}/oauth2/authorize", "https://login.microsoftonline.com/", - Tenant, - $"https://login.microsoftonline.com/{Tenant}" + Tenant }, // v2.0 token endpoint. { $"https://login.microsoftonline.com/{Tenant}/oauth2/v2.0/token", "https://login.microsoftonline.com/", - Tenant, - $"https://login.microsoftonline.com/{Tenant}" + Tenant }, // US Government cloud. { $"https://login.microsoftonline.us/{Tenant}/oauth2/authorize", "https://login.microsoftonline.us/", - Tenant, - $"https://login.microsoftonline.us/{Tenant}" + Tenant }, // Microsoft Azure operated by 21Vianet. { $"https://login.partner.microsoftonline.cn/{Tenant}", "https://login.partner.microsoftonline.cn/", - Tenant, - $"https://login.partner.microsoftonline.cn/{Tenant}" + Tenant }, // Domain-name tenant. { "https://login.microsoftonline.com/contoso.onmicrosoft.com", "https://login.microsoftonline.com/", - "contoso.onmicrosoft.com", - "https://login.microsoftonline.com/contoso.onmicrosoft.com" + "contoso.onmicrosoft.com" }, // Placeholder tenant. { "https://login.microsoftonline.com/common/oauth2/authorize", "https://login.microsoftonline.com/", - "common", - "https://login.microsoftonline.com/common" + "common" }, { "https://login.microsoftonline.com/organizations", "https://login.microsoftonline.com/", - "organizations", - "https://login.microsoftonline.com/organizations" + "organizations" }, { "https://login.microsoftonline.com/consumers", "https://login.microsoftonline.com/", - "consumers", - "https://login.microsoftonline.com/consumers" + "consumers" }, }; + /// + /// Verifies each supported STSURL shape yields the authority URL and first path segment tenant. + /// [Theory] [MemberData(nameof(AuthorityData))] - public void TryParseAuthority_SplitsHostAndTenant( - string authorityUrl, - string expectedHost, - string expectedTenant, - string expectedMsalAuthority) + public void TryParseAuthority_SplitsAuthorityUrlAndTenant( + string stsUrl, + string expectedAuthorityUrl, + string expectedTenant) { Assert.True(ActiveDirectoryAuthenticationProvider.TryParseAuthority( - authorityUrl, - out string host, - out string tenant, - out string msalAuthority)); + stsUrl, + out string authorityUrl, + out string tenant)); - Assert.Equal(expectedHost, host); + Assert.Equal(expectedAuthorityUrl, authorityUrl); Assert.Equal(expectedTenant, tenant); - Assert.Equal(expectedMsalAuthority, msalAuthority); } + /// + /// Verifies invalid STSURLs are rejected without returning partial authority data. + /// [Theory] // A tenant is required; an authority without one cannot yield a usable credential. [InlineData("https://login.microsoftonline.com")] @@ -122,13 +118,11 @@ public void TryParseAuthority_RejectsInvalidAuthority(string authorityUrl) { Assert.False(ActiveDirectoryAuthenticationProvider.TryParseAuthority( authorityUrl, - out string host, - out string tenant, - out string msalAuthority)); + out string parsedAuthorityUrl, + out string tenant)); - Assert.Equal(string.Empty, host); + Assert.Equal(string.Empty, parsedAuthorityUrl); Assert.Equal(string.Empty, tenant); - Assert.Equal(string.Empty, msalAuthority); } /// diff --git a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/Config.cs b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/Config.cs index 1f0587685e..92237bb59e 100644 --- a/src/Microsoft.Data.SqlClient.Extensions/Azure/test/Config.cs +++ b/src/Microsoft.Data.SqlClient.Extensions/Azure/test/Config.cs @@ -38,7 +38,7 @@ internal static class Config internal static bool AdoPool { get; } = false; internal static bool DebugEmit { get; } = false; - internal static bool IntegratedSecuritySupported { get; } = false; + internal static bool EntraIntegratedSupported { get; } = false; internal static bool ManagedIdentitySupported { get; } = false; // @TODO Remove PasswordConnectionString from config; AAD Password auth is deprecated internal static string PasswordConnectionString { get; } = string.Empty; @@ -73,7 +73,7 @@ internal static bool IsAzureSqlServer() => internal static bool OnWindows() => RuntimeInformation.IsOSPlatform(OSPlatform.Windows); internal static bool OnUnix() => OnLinux() || OnMacOS(); - internal static bool SupportsIntegratedSecurity() => IntegratedSecuritySupported; + internal static bool SupportsEntraIntegrated() => EntraIntegratedSupported; internal static bool SupportsManagedIdentity() => ManagedIdentitySupported; internal static bool SupportsSystemAssignedManagedIdentity() => SystemAssignedManagedIdentitySupported; @@ -115,7 +115,7 @@ static Config() // The sample file is copied to the build output directory as // config.jsonc by the TestUtilities project file. // - IntegratedSecuritySupported = GetBool(root, "SupportsIntegratedSecurity"); + EntraIntegratedSupported = GetBool(root, "SupportsEntraIntegrated"); ManagedIdentitySupported = GetBool(root, "ManagedIdentitySupported"); PasswordConnectionString = GetString(root, "AADPasswordConnectionString"); ServicePrincipalId = GetString(root, "AADServicePrincipalId"); @@ -160,7 +160,7 @@ string Base64Encode(string s) Console.WriteLine( $" DebugEmit: {DebugEmit}"); Console.WriteLine( - $" IntegratedSecuritySupported: {IntegratedSecuritySupported}"); + $" EntraIntegratedSupported: {EntraIntegratedSupported}"); Console.WriteLine( $" ManagedIdentitySupported: {ManagedIdentitySupported}"); Console.WriteLine( diff --git a/src/Microsoft.Data.SqlClient/Versions.props b/src/Microsoft.Data.SqlClient/Versions.props index f67997a10f..0e075aa544 100644 --- a/src/Microsoft.Data.SqlClient/Versions.props +++ b/src/Microsoft.Data.SqlClient/Versions.props @@ -31,9 +31,8 @@ - 7.1.0-preview3 + 8.0.0-preview1 @@ -50,13 +49,8 @@ not valid in a file version. Start with the suffix-free package base and append the configured file-version build number, which converts a three-part package version such as 1.2.3-preview1 to 1.2.3.42. - - A revision-enabled pipeline supplies a four-part package base such as 1.2.3.42. That base - is already a complete file version, so the following conditional assignment overrides the - fallback instead of producing an invalid five-part version. --> $(SqlClientPackageVersionBase).$(FileVersionBuildNumber) - $(SqlClientPackageVersionBase) @@ -65,18 +59,24 @@ When a build number is provided, the default version is used as the base of the package and file versions. - If a build suffix is provided, this is appended to the package version. This is meant to be - used by automated pre-release systems to indicate the source of the build. + If a build suffix is provided, it is appended as a prerelease tag. This is meant to be used + by automated pre-release systems to indicate the source of the build. - If a build suffix is not provided, no pre-release tag will be added to package version. If - the default version already has a pre-release tag added to it (eg, "7.0.0-preview1") this - will be retained for the package version, but will be stripped off for the file version - (letters are not allowed in file/assembly versions). This is meant to be used by official - build pipelines to generate production-ready builds. + The build number is then appended to any version carrying a prerelease tag — whether that + tag came from the declared version or from the build suffix — so every automated run + produces a distinct package version (eg, "7.0.0-preview1" becomes + "7.0.0-preview1.26238.3"). A version with no prerelease tag is left exactly as declared, + because released packages are not stamped with a build number. The tag is always stripped + for the file version (letters are not allowed in file/assembly versions). This is meant to + be used by official build pipelines to generate production-ready builds. --> - $(SqlClientNextVersion)-$(BuildSuffix)$(BuildNumber) - $(SqlClientNextVersion) + <_SqlClientCandidateVersion>$(SqlClientNextVersion) + <_SqlClientCandidateVersion Condition="'$(BuildSuffix)' != ''">$(SqlClientNextVersion)-$(BuildSuffix) + + + $(_SqlClientCandidateVersion).$(BuildNumber) + $(_SqlClientCandidateVersion) $(SqlClientNextVersion.Split('-')[0]).$(FileVersionBuildNumber) @@ -105,4 +105,11 @@ $(SqlClientFileVersion.Split('.')[0]).0.0.0 + + + + + diff --git a/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj b/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj index a9474613e6..705182b8d5 100644 --- a/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj +++ b/src/Microsoft.Data.SqlClient/notsupported/Microsoft.Data.SqlClient.csproj @@ -72,6 +72,13 @@ $(NoWarn);CS0618 + + + + <_Parameter1>true + + + + $(RepoRoot)artifacts/ + - $(RepoRoot)artifacts/$(AssemblyName).notsupported/$(ReferenceType)-$(Configuration)/ + $(ArtifactPath)$(AssemblyName).notsupported/$(ReferenceType)-$(Configuration)/ @@ -105,7 +114,7 @@ --> $(RepoRoot)src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj - $(RepoRoot)artifacts/Microsoft.Data.SqlClient.ref/$(ReferenceType)-$(Configuration)/$(TargetFramework)/Microsoft.Data.SqlClient.dll + $(ArtifactPath)Microsoft.Data.SqlClient.ref/$(ReferenceType)-$(Configuration)/$(TargetFramework)/Microsoft.Data.SqlClient.dll +[System.CLSCompliantAttribute(false)] +public enum RegisteredApplication : ushort +{ + /// + Unknown = 0, + /// + EntityFrameworkCore = 1, + /// + SemanticKernel = 2, + /// + ManagementStudio = 3, + /// + SqlManagementObjects = 4, + /// + DataTierApplicationFramework = 5, + /// + SqlToolsService = 6, + /// + AspNetCoreDistributedSqlServerCache = 7, + /// + EntityFramework = 8, + /// + AzureFunctionsSqlExtension = 9, + /// + OrleansAdoNet = 10, + /// + DurableTaskSqlServer = 11, + /// + SqlPackage = 12, + /// + DataApiBuilder = 13 +} + /// public static class SqlClientMetaDataCollectionNames { @@ -1010,6 +1044,9 @@ public SqlConnection() { } public SqlConnection(string connectionString) { } /// public SqlConnection(string connectionString, Microsoft.Data.SqlClient.SqlCredential credential) { } + /// + [System.CLSCompliantAttribute(false)] + public Microsoft.Data.SqlClient.RegisteredApplication RegisteredApplication { get { throw null; } set { } } /// [System.ComponentModel.BrowsableAttribute(false)] @@ -1492,6 +1529,7 @@ public override void Clear() { } #if NETFRAMEWORK /// + [System.ObsoleteAttribute("TransparentNetworkIPResolution has been deprecated and is only supported on .NET Framework. Use MultiSubnetFailover instead.")] [System.ComponentModel.DisplayNameAttribute("Transparent Network IP Resolution")] [System.ComponentModel.RefreshPropertiesAttribute(System.ComponentModel.RefreshProperties.All)] public bool TransparentNetworkIPResolution { get { throw null; } set { } } diff --git a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj index 43aca3f1c1..fad5d71d49 100644 --- a/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj +++ b/src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.csproj @@ -74,6 +74,7 @@ dotnet tool run pwsh -- -NonInteractive + -NoProfile -ExecutionPolicy Unrestricted -Command "$(RepoRoot)tools\intellisense\TrimDocs.ps1 -inputFile '$(DocumentationFile)' -outputFile '$(DocumentationFile)'" diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs index e2e73b55b1..f9410a5c39 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs @@ -1488,11 +1488,8 @@ internal static Exception InvalidMixedUsageOfAccessTokenCallbackAndAuthenticatio internal static Exception InvalidMixedUsageOfAccessTokenCallbackAndIntegratedSecurity() => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidMixedUsageOfAccessTokenCallbackAndIntegratedSecurity)); - internal static Exception InvalidMixedUsageOfAccessTokenAndSspiContextProvider() - => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidMixedUsageOfAccessTokenAndSspiContextProvider)); - - internal static Exception InvalidMixedUsageOfSspiContextProviderAndAccessToken() - => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidMixedUsageOfSspiContextProviderAndAccessToken)); + internal static Exception InvalidMixedUsageOfAccessTokenProperties() + => InvalidOperation(StringsHelper.GetString(Strings.ADP_InvalidMixedUsageOfAccessTokenProperties)); #endregion internal static readonly IntPtr s_ptrZero = IntPtr.Zero; diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs index ec210e407a..be8d9547ef 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs @@ -306,6 +306,12 @@ internal class SqlConnectionInternal : DbConnectionInternal, IDisposable private bool _sessionRecoveryRequested; + /// + /// The middleware application identity of the that caused + /// this physical connection to be created. Reported once, at login. + /// + private readonly RegisteredApplication _registeredApplication; + private int _threadIdOwningParserLock = -1; // @TODO: Rename to indicate this has to do with routing @@ -344,12 +350,14 @@ internal SqlConnectionInternal( IDbConnectionPool pool = null, Func> accessTokenCallback = null, SspiContextProvider sspiContextProvider = null, - ISqlClientMetrics metrics = null) + ISqlClientMetrics metrics = null, + RegisteredApplication registeredApplication = RegisteredApplication.Unknown) : base(metrics) { Debug.Assert(connectionOptions is not null, "null connectionOptions"); ConnectionOptions = connectionOptions; + _registeredApplication = registeredApplication; #if DEBUG if (reconnectSessionData != null) @@ -3063,6 +3071,7 @@ private void Login( login.password = ConnectionOptions.Password; login.applicationName = ConnectionOptions.ApplicationName; login.language = _currentLanguage; + login.appId = _registeredApplication; if (!login.userInstance) { @@ -3973,11 +3982,14 @@ internal override void ResetConnection() if (_fResetConnection) { - // Pooled connections that are enlisted in a transaction must have their transaction - // preserved when resetting the connection state. Otherwise, future uses of the connection - // from the pool will execute outside the transaction, in auto-commit mode. - // https://github.com/dotnet/SqlClient/issues/2970 - _parser.PrepareResetConnection(EnlistedTransaction is not null && Pool is not null); + // Pooled connections that are tied to a transaction must have that transaction + // preserved when resetting the connection state. Otherwise, future uses of the + // connection from the pool will execute outside the transaction, in auto-commit + // mode. See ShouldPreserveTransactionOnReset for the full rationale. + _parser.PrepareResetConnection(ShouldPreserveTransactionOnReset( + isPooled: Pool is not null, + isTransactionRoot: IsTransactionRoot, + hasEnlistedTransaction: EnlistedTransaction is not null)); // Reset dictionary values, since calling reset will not send us env_changes. CurrentDatabase = _originalDatabase; @@ -3985,6 +3997,71 @@ internal override void ResetConnection() } } + /// + /// Decides whether a connection reset must preserve the server-side transaction, i.e. + /// whether ST_RESET_CONNECTION_PRESERVE_TRANSACTION should be sent instead of a + /// plain ST_RESET_CONNECTION. + /// + /// Whether this connection belongs to a pool. + /// + /// Whether this connection is the root of a delegated transaction, i.e. + /// . + /// + /// + /// Whether this connection has a non-null . + /// + /// + /// + /// A pooled connection can be tied to a transaction in two independent ways, and both + /// must be preserved across a reset: + /// + /// + /// + /// It is the root of a delegated transaction: the transaction was delegated to + /// this connection, so it lives on the server session this connection owns. Missing this + /// case resets the server-side transaction out from under System.Transactions, which + /// later breaks the connection while it is being recycled through the pool. + /// See https://github.com/dotnet/SqlClient/issues/4001. + /// + /// + /// It has enlisted in a transaction, so EnlistedTransaction is set. Missing + /// this case causes subsequent uses of the connection to run outside the transaction, in + /// auto-commit mode. See https://github.com/dotnet/SqlClient/issues/2970. + /// + /// + /// + /// These two conditions overlap but neither implies the other. A freshly delegated root + /// has both flags set, because enlistment sets EnlistedTransaction unconditionally. + /// However, once the transaction is no longer Active, + /// DetachCurrentTransactionIfEnded clears EnlistedTransaction while the + /// delegated transaction can still report itself as active. That transient half-state is + /// root-only, and it is exactly the state issue #4001 reproduces in. + /// + /// + /// There is deliberately no server-version guard here. Before + /// https://github.com/dotnet/SqlClient/pull/3019, a delegated root on a pre-2008 server + /// was excluded via IsNonPoolableTransactionRoot, but that property also routed + /// such connections into stasis rather than back into the pool, which is what made + /// suppressing the preserve bit safe. #3019 removed the property, and neither pool puts a + /// poolable root into stasis today, so a version guard would now return the connection to + /// the general pool with its transaction silently reset. That is issue #4001 again. + /// SQL Server 2005 is also outside the supported matrix. + /// + /// + internal static bool ShouldPreserveTransactionOnReset( + bool isPooled, + bool isTransactionRoot, + bool hasEnlistedTransaction) + { + // A connection with no pool is not recycled, so there is nothing to preserve for. + if (!isPooled) + { + return false; + } + + return isTransactionRoot || hasEnlistedTransaction; + } + private void ResolveExtendedServerName(ServerInfo serverInfo, bool aliasLookup, SqlConnectionOptions options) { // @TODO: Invert to save on indentation diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs index a547ce0fd4..6e60f6cdbb 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs @@ -62,12 +62,6 @@ namespace Microsoft.Data.SqlClient.ConnectionPool internal sealed class ChannelDbConnectionPool : IDbConnectionPool, IDisposable { #region Fields - // Limits synchronous operations which depend on async operations on managed - // threads from blocking on all available threads, which would stop async tasks - // from being scheduled and cause deadlocks. Use ProcessorCount/2 as a balance - // between sync and async tasks. - private static SemaphoreSlim _syncOverAsyncSemaphore = new(Math.Max(1, Environment.ProcessorCount / 2)); - /// /// Tracks the number of instances of this class. Used to generate unique IDs for each instance. /// @@ -710,7 +704,9 @@ private void PutConnectionInIdleChannel(DbConnectionInternal connection, bool pr connection.SetReturnedTime(_timeProvider.GetUtcNow().UtcDateTime); } - if (!IsLiveConnection(connection, probeLiveness)) + // Match V1: check token expiry on general checkout, not return. An expired connection + // may remain idle, but will be discarded before it can be reused outside its transaction. + if (!IsLiveConnection(connection, probeLiveness, checkAccessTokenExpiry: false)) { RemoveConnection(connection); return; @@ -939,11 +935,23 @@ public bool TryGetConnection( if (taskCompletionSource is null) { // We're on the caller's thread, so the ambient transaction is directly observable. + Transaction? currentTransaction = ADP.GetCurrentTransaction(); + + // Fast path: when the pool can satisfy the request immediately, do it here rather + // than entering GetInternalConnection, which allocates a Task + // and a timer-backed CancellationTokenSource before it knows whether it will ever + // need to wait. See TryGetPooledConnectionInline. + connection = TryGetPooledConnectionInline(owningObject, currentTransaction); + if (connection is not null) + { + return true; + } + var task = GetInternalConnection( owningObject, async: false, timeout, - ADP.GetCurrentTransaction()); + currentTransaction); // When running synchronously, we are guaranteed that the task is already completed. // We don't need to guard the managed threadpool at this spot because we pass the async flag as false @@ -975,29 +983,21 @@ public bool TryGetConnection( // // The ambient transaction is captured by the caller, on the caller's thread, and handed // to us in the TaskCompletionSource's AsyncState (see SqlConnection.InternalOpenAsync). - // - // We must not read Transaction.Current inside the Task.Run below. A - // TransactionScope created with TransactionScopeAsyncFlowOption.Enabled stores the - // transaction in an AsyncLocal, which does flow onto the pool's worker thread, so that - // would appear to work. But Enabled is not the default: a plain TransactionScope keeps - // the transaction in thread-static storage, which does not flow, and reading - // Transaction.Current on the worker would silently fail to enlist. The WaitHandle pool - // enlists correctly in that case, so this is also a compatibility requirement. - // AsyncState is correct under both options. - // - // This does not make the suppressed-flow pattern work -- the caller's own scope is - // still broken past the first await -- but it keeps the connection in the transaction - // the caller intended rather than silently running outside it. - // - // Note that we deliberately do not assign Transaction.Current on the thread pool - // thread either. That assignment writes to thread-static storage which is *not* unwound - // when the ExecutionContext is restored, so it would outlive this open and be observed - // by unrelated work later scheduled onto the same thread pool thread -- including the - // login-time auto-enlistment that non-pooled connections perform against - // Transaction.Current. The WaitHandle pool can get away with assigning it because it - // processes pending opens on a dedicated non-thread-pool thread. + // Do not read Transaction.Current in the Task.Run below: a plain TransactionScope keeps + // the transaction in thread-static storage, which does not flow to the worker, so the + // enlistment would silently be skipped. Do not assign Transaction.Current there either; + // that storage is not unwound afterwards and would leak into later work on that thread. Transaction? ambientTransaction = taskCompletionSource.Task.AsyncState as Transaction; + // Fast path: return true rather than completing the TaskCompletionSource, so + // InternalOpenAsync takes its sync branch and skips a thread pool dispatch. + DbConnectionInternal? pooled = TryGetPooledConnectionInline(owningObject, ambientTransaction); + if (pooled is not null) + { + connection = pooled; + return true; + } + Task.Run(async () => { if (taskCompletionSource.Task.IsCompleted) @@ -1266,9 +1266,21 @@ _connectionCreationRateLimiter is not null && /// Whether to poll the physical connection to confirm it is still alive. Pass false when /// running on a thread that must not block; the remaining checks are all cheap and local. /// + /// + /// Validate the token before general checkout, but not when returning a connection to the pool. + /// /// Returns true if the connection is live and unexpired, otherwise returns false. - private bool IsLiveConnection(DbConnectionInternal connection, bool probeLiveness = true) + private bool IsLiveConnection(DbConnectionInternal connection, bool probeLiveness = true, bool checkAccessTokenExpiry = true) { + if (checkAccessTokenExpiry && connection.IsAccessTokenExpired) + { + SqlClientEventSource.Log.TryPoolerTraceEvent( + "ChannelDbConnectionPool.IsLiveConnection | INFO | {0}, Connection {1}, will not be reused because its access token has expired or is about to expire.", + Id, + connection.ObjectID); + return false; + } + // Connection has been sitting idle longer than the configured idle timeout. // Checked before the (potentially expensive) liveness probe so an idle-expired // connection is discarded without an SNI round-trip. @@ -1431,6 +1443,66 @@ private void RemoveConnection(DbConnectionInternal connection) return null; } + /// + /// Attempts to satisfy a connection request from connections the pool already holds, + /// without blocking, waiting, or opening a physical connection. + /// + /// + /// The fast path shared by the sync and async entry points of + /// : it tries the transacted store, then the idle channel, and + /// returns null if neither can satisfy the request. It is kept separate from + /// because that method allocates a + /// and a timer-backed even + /// when it completes synchronously. It never calls + /// , which would block on network I/O. + /// + /// The DbConnection that will own this internal connection. + /// The ambient transaction captured on the caller's thread, + /// or null when the caller is not inside a transaction. + /// An activated connection ready to be handed to the caller, or null when the pool + /// cannot satisfy the request without waiting or opening. + /// + /// Propagates any exception from activating or enlisting the connection. The connection is + /// returned to the pool before the exception escapes (see ). + /// + private DbConnectionInternal? TryGetPooledConnectionInline( + DbConnection owningConnection, + Transaction? ambientTransaction) + { + // When automatic enlistment is disabled the connection must never be bound to the + // ambient transaction, so we neither consult the transacted store nor hand the + // transaction to activation. Mirrors GetInternalConnection. + Transaction? transaction = HasTransactionAffinity ? ambientTransaction : null; + + DbConnectionInternal? connection = null; + + // A connection already enlisted in our transaction is always preferred, since reusing + // it avoids promoting the transaction to a distributed one. + if (transaction is not null) + { + connection = GetFromTransactedPool(transaction); + } + + // GetIdleConnection only returns connections that passed IsLiveConnection, and + // GetFromTransactedPool has already probed liveness, so no further validation is + // needed here. GetInternalConnection re-checks after its channel wait because that + // wait can hand back a connection that bypassed both filters. + connection ??= GetIdleConnection(); + + if (connection is null) + { + return null; + } + + // Counted before activation for the same reason as GetInternalConnection: if + // PrepareConnection fails it returns the connection to the pool, which emits the + // matching soft disconnect. Counting after would leave that disconnect unpaired and + // drive the active-soft-connects gauge negative. + Metrics.SoftConnectRequest(); + PrepareConnection(owningConnection, connection, transaction); + return connection; + } + /// /// Gets an internal connection from the pool, either by retrieving an idle connection or opening a new one. /// @@ -1490,7 +1562,7 @@ private async Task GetInternalConnection( { // Skip the liveness/idle/generation gate at the bottom of the loop: // GetFromTransactedPool has already probed liveness, and a transacted - // connection is exempt from idle-timeout, load-balance and + // connection is exempt from token-expiry, idle-timeout, load-balance and // clear-generation eviction because closing it would abort its // (possibly distributed) transaction. break; @@ -1681,30 +1753,10 @@ private void SweepEmancipatedConnections() /// The connection read from the channel. private DbConnectionInternal? ReadChannelSyncOverAsync(CancellationToken cancellationToken) { - // If there are no connections in the channel, then ReadAsync will block until one is available. - // Channels doesn't offer a sync API, so running ReadAsync synchronously on this thread may spawn - // additional new async work items in the managed thread pool if there are no items available in the - // channel. We need to ensure that we don't block all available managed threads with these child - // tasks or we could deadlock. Prefer to block the current user-owned thread, and limit throughput - // to the managed threadpool. - - _syncOverAsyncSemaphore.Wait(cancellationToken); - try - { - ConfiguredValueTaskAwaitable.ConfiguredValueTaskAwaiter awaiter = - _idleChannel.ReadAsync(cancellationToken).ConfigureAwait(false).GetAwaiter(); - using ManualResetEventSlim mres = new ManualResetEventSlim(false, 0); - - // Cancellation happens through the ReadAsync call, which will complete the task. - // Even a failed task will complete and set the ManualResetEventSlim. - awaiter.UnsafeOnCompleted(() => mres.Set()); - mres.Wait(CancellationToken.None); - return awaiter.GetResult(); - } - finally - { - _syncOverAsyncSemaphore.Release(); - } + // Channel has no blocking read. Block on the Task rather than an opaque primitive: the + // idle channel is created without AllowSynchronousContinuations, so the completing + // continuation is queued, and Task blocking lets the thread pool inject a worker to run it. + return _idleChannel.ReadAsync(cancellationToken).AsTask().GetAwaiter().GetResult(); } /// diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs index afaa827fce..887bcd5c35 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs @@ -900,12 +900,8 @@ public bool TryGetConnection(DbConnection owningObject, TaskCompletionSource {0}, Pool is shutting down; abandoning wait.", Id); - if (waitResult == SEMAPHORE_HANDLE || waitResult == WAIT_ABANDONED + SEMAPHORE_HANDLE) - { - try - { - _waitHandles.PoolSemaphore.Release(1); - } - catch (SemaphoreFullException) - { - // Pool semaphore was already saturated by Shutdown's bulk release; safe to ignore. - } - } - Interlocked.Decrement(ref _waitCount); - connection = null; - return false; - } - // From the WaitAny docs: "If more than one object became signaled during // the call, this is the array index of the signaled object with the // smallest index value of all the signaled objects." This is important @@ -1597,37 +1565,17 @@ public void Shutdown() } State = ShuttingDown; - // Dispose all background timers so they no longer schedule new work. - // Note that any timer callback already in flight may still observe State == ShuttingDown - // and short-circuit (see CleanupCallback / ErrorCallback). + // Stop maintenance, but let admitted requests finish. Their connections are + // destroyed by DeactivateObject when returned to this retired pool. Timer cleanup = Interlocked.Exchange(ref _cleanupTimer, null); cleanup?.Dispose(); - _errorState.Dispose(); - - // Wake any threads parked in WaitHandle.WaitAny by releasing as many semaphore - // slots as there are recorded waiters. Using _waitCount (rather than MaxPoolSize) - // avoids ArgumentOutOfRangeException when MaxPoolSize == 0 (unlimited) and ensures - // we wake every parked waiter even when _waitCount exceeds MaxPoolSize. Waiters - // observe State is not Running after wake-up and bail. - int waitersToWake = Volatile.Read(ref _waitCount); - if (waitersToWake > 0) - { - try - { - _waitHandles.PoolSemaphore.Release(waitersToWake); - } - catch (SemaphoreFullException) - { - // Semaphore already saturated; nothing to do. - } - } + // Keep the cached error and its expiry timer available to admitted waiters. + // Disposing the error state here leaves ErrorEvent signaled without an error. - // Reuse Clear() to doom every connection (including active checked-out ones), drain - // both idle stacks, and reclaim emancipated objects. Active connections destroy - // themselves on return either via the doom flag or via DeactivateObject's - // State == ShuttingDown branch. - Clear(); + // Leave Clear() to the factory's explicit-clear or deferred-pruning path. + // Shutdown can run under the pool-group lock, where reclamation and connection + // disposal must not be added before the pool is queued for release. } // TransactionEnded merely provides the plumbing for DbConnectionInternal to access the transacted pool diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Diagnostics/SqlDiagnosticListener.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Diagnostics/SqlDiagnosticListener.cs index 40e4bb63d9..0777a527b3 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Diagnostics/SqlDiagnosticListener.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Diagnostics/SqlDiagnosticListener.cs @@ -5,6 +5,7 @@ using System; using System.Data; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; #if NET using System.Reflection; @@ -77,7 +78,7 @@ public void WriteCommandAfter( return; } - Write( + WriteEvent( SqlClientCommandAfter.Name, new SqlClientCommandAfter( operationId, @@ -104,7 +105,7 @@ public Guid WriteCommandBefore( Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientCommandBefore.Name, new SqlClientCommandBefore( operationId, @@ -132,7 +133,7 @@ public void WriteCommandError( return; } - Write( + WriteEvent( SqlClientCommandError.Name, new SqlClientCommandError ( @@ -159,7 +160,7 @@ public void WriteConnectionCloseAfter( return; } - Write( + WriteEvent( SqlClientConnectionCloseAfter.Name, new SqlClientConnectionCloseAfter ( @@ -182,7 +183,7 @@ public Guid WriteConnectionCloseBefore(SqlConnection sqlConnection, [CallerMembe Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientConnectionCloseBefore.Name, new SqlClientConnectionCloseBefore ( @@ -211,7 +212,7 @@ public void WriteConnectionCloseError( return; } - Write( + WriteEvent( SqlClientConnectionCloseError.Name, new SqlClientConnectionCloseError ( @@ -237,7 +238,7 @@ public void WriteConnectionOpenAfter( return; } - Write( + WriteEvent( SqlClientConnectionOpenAfter.Name, new SqlClientConnectionOpenAfter ( @@ -261,7 +262,7 @@ public Guid WriteConnectionOpenBefore(SqlConnection sqlConnection, [CallerMember Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientConnectionOpenBefore.Name, new SqlClientConnectionOpenBefore ( @@ -288,7 +289,7 @@ public void WriteConnectionOpenError( return; } - Write( + WriteEvent( SqlClientConnectionOpenError.Name, new SqlClientConnectionOpenError ( @@ -316,7 +317,7 @@ public void WriteTransactionCommitAfter( return; } - Write( + WriteEvent( SqlClientTransactionCommitAfter.Name, new SqlClientTransactionCommitAfter ( @@ -344,7 +345,7 @@ public Guid WriteTransactionCommitBefore( Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientTransactionCommitBefore.Name, new SqlClientTransactionCommitBefore ( @@ -374,7 +375,7 @@ public void WriteTransactionCommitError( return; } - Write( + WriteEvent( SqlClientTransactionCommitError.Name, new SqlClientTransactionCommitError ( @@ -403,7 +404,7 @@ public void WriteTransactionRollbackAfter( return; } - Write( + WriteEvent( SqlClientTransactionRollbackAfter.Name, new SqlClientTransactionRollbackAfter ( @@ -433,7 +434,7 @@ public Guid WriteTransactionRollbackBefore( Guid operationId = Guid.NewGuid(); - Write( + WriteEvent( SqlClientTransactionRollbackBefore.Name, new SqlClientTransactionRollbackBefore ( @@ -465,7 +466,7 @@ public void WriteTransactionRollbackError( return; } - Write( + WriteEvent( SqlClientTransactionRollbackError.Name, new SqlClientTransactionRollbackError ( @@ -481,6 +482,17 @@ public void WriteTransactionRollbackError( ); } +#if NET + [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026", + Justification = "SqlClient does no reflection here; subscribers that reflect over payloads are responsible for their own reflection.")] + // The annotation on T is required by Write and keeps payload public properties for subscribers that reflect. + private void WriteEvent<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties)] T>(string name, T payload) => + Write(name, payload); +#else + private void WriteEvent(string name, T payload) => + Write(name, payload); +#endif + #if NET private void SqlDiagnosticListener_UnloadingAssemblyLoadContext(AssemblyLoadContext obj) => Dispose(); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs index de40956784..b5254cde75 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniProxy.netcore.cs @@ -275,6 +275,17 @@ private static SniNpHandle CreateNpHandle(DataSource details, TimeoutTimer timeo SniCommon.ReportSNIError(SniProviders.NP_PROV, 0, SniCommon.MultiSubnetFailoverWithNonTcpProtocol, Strings.SNI_ERROR_49); return null; } + + // Final safeguard: never hand a pipe path whose host component contains a colon to the + // OS. DataSource transcribes IPv6 literals during parsing, so anything still holding a + // colon here is malformed. See DataSource.GetUncCompatibleHostName for details. + if (string.IsNullOrEmpty(details.PipeHostName) || details.PipeHostName.IndexOf(':') != -1) + { + SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniProxy), EventType.ERR, "Invalid host name '{0}' for Named Pipes.", details.PipeHostName); + SniCommon.ReportSNIError(SniProviders.NP_PROV, 0, SniCommon.InvalidConnStringError, Strings.SNI_ERROR_25); + return null; + } + return new SniNpHandle(details.PipeHostName, details.PipeName, timeout, tlsFirst, hostNameInCertificate, serverCertificateFilename); } @@ -339,6 +350,7 @@ internal class DataSource private const string DefaultPipeName = "sql\\query"; private const string InstancePrefix = "MSSQL$"; private const string PathSeparator = "\\"; + private const string IPv6LiteralHostSuffix = ".ipv6-literal.net"; internal enum Protocol { TCP, NP, None, Admin }; @@ -632,7 +644,7 @@ private bool InferNamedPipesInformation() { // NamedPipeClientStream object will create the network path using PipeHostName and PipeName // and can be seen in its _normalizedPipePath variable in the format \\servername\pipe\MSSQL$\sql\query - PipeHostName = ServerName = tokensByBackSlash[0]; + ServerName = tokensByBackSlash[0]; PipeName = $"{InstancePrefix}{tokensByBackSlash[1]}{PathSeparator}{DefaultPipeName}"; } else @@ -643,10 +655,23 @@ private bool InferNamedPipesInformation() } else { - PipeHostName = ServerName = _dataSourceAfterTrimmingProtocol; + ServerName = _dataSourceAfterTrimmingProtocol; PipeName = SniNpHandle.DefaultPipePath; } + // An IPv6 literal must be transcribed before it can appear in a UNC pipe path, + // and ServerName must drop any brackets because it feeds DNS resolution and SPN + // construction. See GetUncCompatibleHostName and NormalizeHostName for details. + PipeHostName = GetUncCompatibleHostName(ServerName); + if (PipeHostName is null) + { + SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniProxy), EventType.ERR, "Invalid host name '{0}' for Named Pipes.", ServerName); + ReportSNIError(SniProviders.NP_PROV); + return false; + } + + ServerName = NormalizeHostName(ServerName); + InferLocalServerName(); return true; } @@ -700,10 +725,22 @@ private bool InferNamedPipesInformation() InstanceName = PipeToken + PipeName; } - ServerName = IsLocalHost(host) ? Environment.MachineName : host; + // An IPv6 literal must be transcribed before it can appear in a UNC pipe path. + // See GetUncCompatibleHostName for details. + string uncHost = GetUncCompatibleHostName(host); + if (uncHost is null) + { + SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniProxy), EventType.ERR, "Invalid host name '{0}' for Named Pipes.", host); + ReportSNIError(SniProviders.NP_PROV); + return false; + } + + // ServerName drops any brackets because it feeds DNS resolution and SPN + // construction, neither of which accepts the bracketed spelling. + ServerName = IsLocalHost(host) ? Environment.MachineName : NormalizeHostName(host); // Pipe hostname is the hostname after leading \\ which should be passed down as is to open Named Pipe. // For Named Pipes the ServerName makes sense for SPN creation only. - PipeHostName = host; + PipeHostName = uncHost; } catch (UriFormatException) { @@ -729,6 +766,99 @@ private bool InferNamedPipesInformation() private static bool IsLocalHost(string serverName) => ".".Equals(serverName) || "(local)".Equals(serverName) || "localhost".Equals(serverName); + + /// + /// Attempts to interpret a host name as an IPv6 literal, accepting the bracketed form + /// ([::1]) that users may carry over from URL syntax. + /// + private static bool TryParseIPv6Literal(string hostName, out IPAddress address) + { + address = null; + + // A colon is the only character that can make a host name an IPv6 literal, so anything + // without one (host names, IPv4 literals, already-transcribed names) is not a candidate. + if (string.IsNullOrEmpty(hostName) || hostName.IndexOf(':') == -1) + { + return false; + } + + ReadOnlySpan literal = hostName.AsSpan(); + if (literal.Length > 2 && literal[0] == '[' && literal[literal.Length - 1] == ']') + { + literal = literal.Slice(1, literal.Length - 2); + } + + return IPAddress.TryParse(literal, out address) && + address.AddressFamily == AddressFamily.InterNetworkV6; + } + + /// + /// Returns the canonical form of a host name: a bracketed IPv6 literal is unwrapped to its + /// unbracketed form, and every other host name is returned unchanged. + /// + /// + /// feeds DNS resolution and SPN construction, neither of which + /// accepts the bracketed spelling, so the brackets must be dropped before it is used there. + /// + internal static string NormalizeHostName(string hostName) => + TryParseIPv6Literal(hostName, out IPAddress address) ? address.ToString() : hostName; + + /// + /// Converts a host name into a form that can legally appear as the host component of a UNC + /// pipe path (\\host\pipe\sql\query), returning if no such + /// form exists. + /// + /// + /// A UNC host component may never contain a colon, so an IPv6 literal such as ::1 + /// cannot be used directly. Passing one through anyway composes a malformed path like + /// \\::1\pipe\sql\query, which sends the SMB redirector into an SMB session setup + /// whose SPNEGO/NegoEx target name embeds the IPv6 literal; that can fault LSASS on Windows + /// and force a reboot. See https://github.com/dotnet/SqlClient/issues/4523. + /// + /// Windows defines a transcription for exactly this case: replace each : with + /// - and each % (zone index) with s, then append + /// .ipv6-literal.net. For example 2001:db8::1 becomes + /// 2001-db8--1.ipv6-literal.net. See + /// https://learn.microsoft.com/openspecs/windows_protocols/ms-dtyp/62e862f4-2a51-452e-8eeb-dc4ff5ee33cc. + /// + /// Host names without a colon (including IPv4 literals and already-transcribed + /// .ipv6-literal.net names) are returned unchanged. A colon-bearing host name that is + /// not a parseable IPv6 literal has no UNC form and is rejected. + /// + internal static string GetUncCompatibleHostName(string hostName) + { + if (string.IsNullOrEmpty(hostName)) + { + return null; + } + + if (hostName.IndexOf(':') == -1) + { + return hostName; + } + + if (!TryParseIPv6Literal(hostName, out IPAddress address)) + { + return null; + } + + string literal = address.ToString(); + return string.Create(literal.Length + IPv6LiteralHostSuffix.Length, literal, + static (destination, value) => + { + for (int i = 0; i < value.Length; i++) + { + destination[i] = value[i] switch + { + ':' => '-', + '%' => 's', + _ => value[i] + }; + } + + IPv6LiteralHostSuffix.AsSpan().CopyTo(destination.Slice(value.Length)); + }); + } } } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/RegisteredApplication.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/RegisteredApplication.cs new file mode 100644 index 0000000000..54aba7eaf8 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/RegisteredApplication.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#nullable enable + +namespace Microsoft.Data.SqlClient; + +/// +[System.CLSCompliant(false)] +public enum RegisteredApplication : ushort +{ + /// + Unknown = 0, + /// + EntityFrameworkCore = 1, + /// + SemanticKernel = 2, + /// + ManagementStudio = 3, + /// + SqlManagementObjects = 4, + /// + DataTierApplicationFramework = 5, + /// + SqlToolsService = 6, + /// + AspNetCoreDistributedSqlServerCache = 7, + /// + EntityFramework = 8, + /// + AzureFunctionsSqlExtension = 9, + /// + OrleansAdoNet = 10, + /// + DurableTaskSqlServer = 11, + /// + SqlPackage = 12, + /// + DataApiBuilder = 13 +} diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs index 96eadb9d56..df556a49ba 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Reliability/SqlConfigurableRetryLogicLoader.cs @@ -50,14 +50,6 @@ public SqlConfigurableRetryLogicLoader( string cnnSectionName = SqlConfigurableRetryConnectionSection.Name, string cmdSectionName = SqlConfigurableRetryCommandSection.Name) { - #if NET - // Just only one subscription to this event is required. - // This class isn't supposed to be called more than one time; - // SqlConfigurableRetryLogicManager manages a single instance of this class. - System.Runtime.Loader.AssemblyLoadContext.Default.Resolving -= Default_Resolving; - System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += Default_Resolving; - #endif - AssignProviders(connectionRetryConfigs == null ? null : CreateRetryLogicProvider(cnnSectionName, connectionRetryConfigs), commandRetryConfigs == null ? null : CreateRetryLogicProvider(cmdSectionName, commandRetryConfigs)); } @@ -119,44 +111,70 @@ private static SqlRetryLogicBaseProvider ResolveRetryLogicProvider(string config throw new ArgumentNullException(nameof(retryMethod), StringsHelper.GetString(Strings.SQLCR_RetryMethodNullOrEmpty)); } - Type type = null; - try - { - // Resolve a Type object from the given type name - // Different implementation in .NET Framework & .NET Core - type = LoadType(configurableRetryType); - } - catch (Exception e) - { - // Try to use 'SqlConfigurableRetryFactory' as a default type to discover retry methods - // if there is a problem, resolve using the 'configurableRetryType' type. - type = typeof(SqlConfigurableRetryFactory); - SqlClientEventSource.Log.TryTraceEvent(" Unable to load the '{2}' type; Trying to use the internal `{3}` type: {4}", - TypeName, methodName, configurableRetryType, type.FullName, e); - } + Type type; - // Run the function by using the resolved values to get the SqlRetryLogicBaseProvider object - try + // Whitespace is treated the same as unset: there is no type to resolve, so the + // resolving handler must not be installed for it. + bool customRetryTypeConfigured = !string.IsNullOrWhiteSpace(configurableRetryType); + + // Keep the handler subscribed across both type resolution and provider construction. + // Invoking the configured type's constructor and retry method can load that assembly's + // private dependencies after LoadType has returned. + using (AssemblyResolutionSubscription subscription = new(customRetryTypeConfigured)) { - // Create an instance from the discovered type by its default constructor - object result = CreateInstance(type, retryMethod, option); + if (!customRetryTypeConfigured) + { + // No custom retry logic type was configured, so there is nothing to resolve and + // the built-in factory is used to discover the requested retry method. + type = typeof(SqlConfigurableRetryFactory); + SqlClientEventSource.Log.TryTraceEvent(" No custom retry logic type is configured; Using the internal `{2}` type.", + TypeName, methodName, type.FullName); + } + else + { + try + { + // Resolve a Type object from the given type name + // Different implementation in .NET Framework & .NET Core + type = LoadType(configurableRetryType); + } + catch (Exception e) + { + // The custom type will not be constructed, so the built-in fallback no + // longer needs the custom assembly resolution handler. + subscription.Dispose(); + + // Try to use 'SqlConfigurableRetryFactory' as a default type to discover retry methods + // if there is a problem, resolve using the 'configurableRetryType' type. + type = typeof(SqlConfigurableRetryFactory); + SqlClientEventSource.Log.TryTraceEvent(" Unable to load the '{2}' type; Trying to use the internal `{3}` type: {4}", + TypeName, methodName, configurableRetryType, type.FullName, e); + } + } - if (result is SqlRetryLogicBaseProvider provider) + // Run the function by using the resolved values to get the SqlRetryLogicBaseProvider object + try { - SqlClientEventSource.Log.TryTraceEvent(" The created instace is a {2} type.", - TypeName, methodName, typeof(SqlRetryLogicBaseProvider).FullName); - provider.Retrying += OnRetryingEvent; - return provider; + // Create an instance from the discovered type by its default constructor + object result = CreateInstance(type, retryMethod, option); + + if (result is SqlRetryLogicBaseProvider provider) + { + SqlClientEventSource.Log.TryTraceEvent(" The created instace is a {2} type.", + TypeName, methodName, typeof(SqlRetryLogicBaseProvider).FullName); + provider.Retrying += OnRetryingEvent; + return provider; + } + } + catch (Exception e) + { + // In order to invoke a function dynamically, any type of exception can occur here; + // The main exception and its stack trace will be accessible through the inner exception. + // i.e: Opening a connection or executing a command while invoking a function + // runs the application to the `TargetInvocationException`. + // And using an isolated zone like a specific AppDomain results in an infinite loop. + throw new InvalidOperationException(StringsHelper.GetString(Strings.SQLCR_RetryMethodException, type.FullName, retryMethod), e); } - } - catch (Exception e) - { - // In order to invoke a function dynamically, any type of exception can occur here; - // The main exception and its stack trace will be accessible through the inner exception. - // i.e: Opening a connection or executing a command while invoking a function - // runs the application to the `TargetInvocationException`. - // And using an isolated zone like a specific AppDomain results in an infinite loop. - throw new InvalidOperationException(StringsHelper.GetString(Strings.SQLCR_RetryMethodException, type.FullName, retryMethod), e); } SqlClientEventSource.Log.TryTraceEvent(" Unable to resolve a valid provider; Returns `null`.", TypeName, methodName); @@ -334,13 +352,53 @@ private static ICollection SplitErrorNumberList(string list) } #region Type Resolution - + + internal sealed class AssemblyResolutionSubscription : IDisposable + { + #if NET + private bool _isSubscribed; + #endif + + internal AssemblyResolutionSubscription(bool subscribe) + { + #if NET + if (subscribe) + { + AssemblyLoadContext.Default.Resolving += Default_Resolving; + _isSubscribed = true; + } + #endif + } + + public void Dispose() + { + #if NET + if (_isSubscribed) + { + AssemblyLoadContext.Default.Resolving -= Default_Resolving; + _isSubscribed = false; + } + #endif + } + } + #if NET + /// + /// The directory that user-supplied configurable retry logic assemblies are probed from. + /// + /// + /// This is deliberately the application base directory rather than the current working + /// directory. The working directory is ambient process state that can be changed at any + /// time and is not necessarily related to where the application's binaries live, so + /// probing it can load assemblies from an unintended and untrusted location. + /// + private static string ProbingDirectory => AppContext.BaseDirectory; + private static Assembly AssemblyResolver(AssemblyName arg) { string methodName = nameof(AssemblyResolver); - string fullPath = MakeFullPath(Environment.CurrentDirectory, arg.Name); + string fullPath = MakeFullPath(ProbingDirectory, arg.Name); SqlClientEventSource.Log.TryTraceEvent(" Looking for '{2}' assembly by '{3}' full path." , TypeName, methodName, arg, fullPath); @@ -348,13 +406,19 @@ private static Assembly AssemblyResolver(AssemblyName arg) } /// - /// Load assemblies on request. + /// Load dependencies of a user-supplied configurable retry logic assembly on request. /// + /// + /// This handler is only subscribed while a configured retry logic provider is being + /// resolved and constructed, and only when a custom retry logic type has been configured. + /// It must not remain subscribed to after that, + /// because doing so changes assembly resolution behavior for the entire application. + /// private static Assembly Default_Resolving(AssemblyLoadContext arg1, AssemblyName arg2) { string methodName = nameof(Default_Resolving); - string target = MakeFullPath(Environment.CurrentDirectory, arg2.Name); + string target = MakeFullPath(ProbingDirectory, arg2.Name); SqlClientEventSource.Log.TryTraceEvent(" Looking for '{2}' assembly that is requested by '{3}' ALC from '{4}' path." , TypeName, methodName, arg2, arg1, target); @@ -371,7 +435,10 @@ private static Type LoadType(string fullyQualifiedName) string methodName = nameof(LoadType); SqlClientEventSource.Log.TryTraceEvent(" Entry point.", TypeName, methodName); - var result = Type.GetType(fullyQualifiedName, AssemblyResolver, TypeResolver); + Type result; + + result = Type.GetType(fullyQualifiedName, AssemblyResolver, TypeResolver); + if (result != null) { SqlClientEventSource.Log.TryTraceEvent(" The '{2}' type is resolved.", diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlClientDriverProperties.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlClientDriverProperties.cs new file mode 100644 index 0000000000..07378821e4 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlClientDriverProperties.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +#nullable enable + +namespace Microsoft.Data.SqlClient; + +/// +/// Driver-owned feature flags reported in the Driver Properties part of the +/// USERAGENT login feature extension payload. +/// +/// +/// This part is driver-owned, so its meaning is defined entirely by +/// Microsoft.Data.SqlClient and carries no cross-driver contract. Other +/// drivers use the same part for their own purposes. +/// +[Flags] +internal enum SqlClientDriverProperties : ulong +{ + /// No tracked features are enabled. + None = 0, + + /// + /// The connection pool V2 implementation + /// (Switch.Microsoft.Data.SqlClient.UseConnectionPoolV2) is + /// enabled. + /// + ConnectionPoolV2 = 1 +} + +/// +/// Resolves the flags that describe +/// how this process is configured. +/// +internal static class SqlClientDriverPropertiesResolver +{ + /// + /// The flags describing the current process. + /// + /// + /// The flags are sourced from process-wide switches, so this is stable + /// for the life of the process. + /// + internal static SqlClientDriverProperties Current => + Resolve(LocalAppContextSwitches.UseConnectionPoolV2); + + /// + /// Maps the process configuration to the flags that describe it. + /// + /// + /// Whether the connection pool V2 implementation is enabled. + /// + /// + /// The flags describing the supplied configuration. + /// + /// + /// The mapping is kept separate from because the + /// switches it reads are cached for the life of the process, which makes + /// them impractical to vary in a test. + /// + internal static SqlClientDriverProperties Resolve(bool useConnectionPoolV2) + { + SqlClientDriverProperties properties = SqlClientDriverProperties.None; + + if (useConnectionPoolV2) + { + properties |= SqlClientDriverProperties.ConnectionPoolV2; + } + + return properties; + } +} diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs index 294310a08f..c90ac4520d 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs @@ -1306,7 +1306,7 @@ public void ResetCommandTimeout() #region Internal Methods // @TODO: This is only called by SqlCommandBuilder, it should live there. EXCEPT for the one call to ValidateCommand and setting _parameters at the end. Is that really necessary? - // @TODO: This also an crazy long method. + // @TODO: This method is also excessively long. internal void DeriveParameters() { switch (CommandType) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnection.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnection.cs index b8b693132e..66b8df4564 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnection.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnection.cs @@ -262,6 +262,7 @@ private SqlConnection(SqlConnection connection) _accessToken = connection._accessToken; _accessTokenCallback = connection._accessTokenCallback; + RegisteredApplication = connection.RegisteredApplication; // CopyFrom retains the source PoolGroup, and therefore the source ConnectionPoolKey. // The provider must be copied along with it, otherwise the clone would authenticate @@ -381,6 +382,34 @@ public static void RegisterColumnEncryptionKeyStoreProviders(IDictionary + [CLSCompliant(false)] + public RegisteredApplication RegisteredApplication + { + get; + set + { + DbConnectionInternal connectionInternal = InnerConnection; + bool canSet = connectionInternal.AllowSetConnectionString; + if (canSet) + { + // Reserve the closed state so Open cannot read the old value while this setter + // publishes the new one. + canSet = SetInnerConnectionFrom(DbConnectionClosedBusy.SingletonInstance, connectionInternal); + if (canSet) + { + field = value; + SetInnerConnectionTo(connectionInternal); + } + } + + if (!canSet) + { + throw ADP.OpenConnectionPropertySet(nameof(RegisteredApplication), connectionInternal.State); + } + } + } + /// public void RegisterColumnEncryptionKeyStoreProvidersOnConnection(IDictionary customProviders) { @@ -1199,7 +1228,7 @@ private void CheckAndThrowOnInvalidCombinationOfConnectionOptionAndAccessToken(S if (_sspiContextProvider != null) { - throw ADP.InvalidMixedUsageOfAccessTokenAndSspiContextProvider(); + throw ADP.InvalidMixedUsageOfAccessTokenProperties(); } } @@ -1226,7 +1255,7 @@ private void CheckAndThrowOnInvalidCombinationOfConnectionOptionAndAccessTokenCa if (_sspiContextProvider != null) { - throw ADP.InvalidMixedUsageOfAccessTokenAndSspiContextProvider(); + throw ADP.InvalidMixedUsageOfAccessTokenProperties(); } } @@ -1238,7 +1267,7 @@ private void CheckAndThrowOnInvalidCombinationOfConnectionOptionAndSspiContextPr { if (_accessToken != null || _accessTokenCallback != null) { - throw ADP.InvalidMixedUsageOfSspiContextProviderAndAccessToken(); + throw ADP.InvalidMixedUsageOfAccessTokenProperties(); } } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionFactory.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionFactory.cs index c3b7bfd13b..e5c567d117 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionFactory.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionFactory.cs @@ -712,7 +712,8 @@ protected virtual DbConnectionInternal CreateConnection( redirectedUserInstance: false, applyTransientFaultHandling: applyTransientFaultHandling, sspiContextProvider: key.SspiContextProvider, - metrics: Metrics); + metrics: Metrics, + registeredApplication: sqlOwningConnection?.RegisteredApplication ?? RegisteredApplication.Unknown); using (sseConnection) { // NOTE: Retrieve here. This user instance name will be @@ -774,7 +775,8 @@ protected virtual DbConnectionInternal CreateConnection( pool, key.AccessTokenCallback, key.SspiContextProvider, - metrics: Metrics); + metrics: Metrics, + registeredApplication: sqlOwningConnection?.RegisteredApplication ?? RegisteredApplication.Unknown); } private static DbConnectionPoolGroupOptions CreateConnectionPoolGroupOptions(SqlConnectionOptions connectionOptions) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs index 4038dbb295..bb4871ed4a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionStringBuilder.cs @@ -411,7 +411,9 @@ private object GetAt(Keywords index) return ConnectionReset; #pragma warning restore 618 case Keywords.TransparentNetworkIPResolution: +#pragma warning disable 618 // Obsolete properties return TransparentNetworkIPResolution; +#pragma warning restore 618 case Keywords.NetworkLibrary: return NetworkLibrary; #endif @@ -1083,7 +1085,9 @@ public override object this[string keyword] NetworkLibrary = ConvertToString(value); break; case Keywords.TransparentNetworkIPResolution: +#pragma warning disable 618 // Obsolete properties TransparentNetworkIPResolution = ConvertToBoolean(value); +#pragma warning restore 618 break; #endif default: @@ -1879,6 +1883,7 @@ public bool ConnectionReset } /// + [Obsolete("TransparentNetworkIPResolution has been deprecated and is only supported on .NET Framework. Use MultiSubnetFailover instead.")] [DisplayName(DbConnectionStringKeywords.TransparentNetworkIpResolution)] [ResCategory(nameof(Strings.DataCategory_Source))] [ResDescription(nameof(Strings.DbConnectionString_TransparentNetworkIPResolution))] diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs index eba61969c3..55b3363261 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.DataTypes.cs @@ -13,7 +13,7 @@ namespace Microsoft.Data.SqlClient; internal sealed partial class SqlMetaDataFactory { - private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet) + private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet, bool jsonTypeSupported) { DataTable dataTypesDataTable = CreateDataTypesDataTable(); @@ -58,8 +58,10 @@ private static void LoadDataTypesDataTables(DataSet metaDataCollectionsDataSet) minimumVersion: "10.00.000.0"); AddLongStringOrBinaryType(SqlDbType.Xml); - AddLongStringOrBinaryType(SqlDbTypeExtensions.Json, literalPrefix: "'", literalSuffix: "'", - minimumVersion: "17.00.000.0"); + if (jsonTypeSupported) + { + AddLongStringOrBinaryType(SqlDbTypeExtensions.Json, literalPrefix: "'", literalSuffix: "'"); + } AddLongStringOrBinaryType(SqlDbType.Text, literalPrefix: "'", literalSuffix: "'"); AddLongStringOrBinaryType(SqlDbType.NText, literalPrefix: "N'", literalSuffix: "'"); AddLongStringOrBinaryType(SqlDbType.Image, literalPrefix: "0x"); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs index bea2fa7e41..83eb6d857a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlMetaDataFactory.cs @@ -39,6 +39,7 @@ internal sealed partial class SqlMetaDataFactory : IDisposable private readonly DataSet _collectionDataSet; private readonly string _serverVersion; + private readonly bool _jsonTypeSupported; public SqlMetaDataFactory(Stream xmlStream, ConnectionCapabilities connectionCapabilities) { @@ -47,6 +48,7 @@ public SqlMetaDataFactory(Stream xmlStream, ConnectionCapabilities connectionCap ADP.CheckArgumentNull(connectionCapabilities.ServerVersion, nameof(connectionCapabilities.ServerVersion)); _serverVersion = connectionCapabilities.ServerVersion; + _jsonTypeSupported = connectionCapabilities.JsonType; _collectionDataSet = LoadDataSetFromXml(xmlStream); } @@ -711,7 +713,7 @@ private DataSet LoadDataSetFromXml(Stream XmlStream) Locale = CultureInfo.InvariantCulture }; - LoadDataTypesDataTables(metaDataCollectionsDataSet); + LoadDataTypesDataTables(metaDataCollectionsDataSet, _jsonTypeSupported); XmlReaderSettings settings = new() { diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs index d8c904f1e0..92e40cc292 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParser.cs @@ -1356,12 +1356,15 @@ internal void TdsLogin( } int feOffset = length; + // Capture the payload once so the length reserved below and the + // bytes written by WriteLoginData can never disagree. + ReadOnlyMemory userAgent = UserAgent.GetUcs2Bytes(rec.appId); // calculate and reserve the required bytes for the featureEx length = ApplyFeatureExData( requestedFeatures, recoverySessionData, fedAuthFeatureExtensionData, - UserAgent.Ucs2Bytes, + userAgent, useFeatureExt, length ); @@ -1380,7 +1383,8 @@ internal void TdsLogin( length, feOffset, clientInterfaceName, - sspiWriter is { } ? sspiWriter.WrittenSpan : ReadOnlySpan.Empty); + sspiWriter is { } ? sspiWriter.WrittenSpan : ReadOnlySpan.Empty, + userAgent); } finally { @@ -9261,7 +9265,8 @@ private void WriteLoginData(SqlLogin rec, int length, int featureExOffset, string clientInterfaceName, - ReadOnlySpan outSSPI) + ReadOnlySpan outSSPI, + ReadOnlyMemory userAgent) { try { @@ -9521,7 +9526,7 @@ private void WriteLoginData(SqlLogin rec, requestedFeatures, recoverySessionData, fedAuthFeatureExtensionData, - UserAgent.Ucs2Bytes, + userAgent, useFeatureExt, length, true @@ -10428,7 +10433,10 @@ private Task TDSExecuteRPCAddParameter(TdsParserStateObject stateObj, SqlParamet // If Precision is specified, verify value precision vs param precision if (precision != 0) { - if (precision < adjustedValue.Precision) + // Precision metadata can overstate zero's required digits. + // Compare magnitudes to recognize negative zero as well. + if (precision < adjustedValue.Precision && + (SqlDecimal.Abs(adjustedValue) != new SqlDecimal(0)).IsTrue) { throw ADP.ParameterValueOutOfRange(adjustedValue); } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs index 6bcc3f2d41..fa59185462 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs @@ -124,6 +124,7 @@ internal sealed class SqlLogin internal bool readOnlyIntent = false; // read-only intent internal SqlCredential credential; // user id and password in SecureString internal SecureString newSecurePassword; + internal RegisteredApplication appId = RegisteredApplication.Unknown; // middleware application identity } #nullable enable diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/UserAgent.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/UserAgent.cs index d8820c3ce9..fc7b8b403f 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/UserAgent.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/UserAgent.cs @@ -3,8 +3,10 @@ // See the LICENSE file in the project root for more information. using System; +using System.Globalization; using System.Runtime.InteropServices; using System.Text; +using System.Threading; #nullable enable @@ -31,9 +33,15 @@ internal static class UserAgent /// never larger than 256 characters. /// /// - /// The format is pipe ('|') delimited into 7 parts: + /// The format is pipe ('|') delimited into 9 parts: /// - /// 1|MS-MDS|{Driver Version}|{Arch}|{OS Type}|{OS Info}|{Runtime Info} + /// 2|MS-MDS|{Driver Version}|{Arch}|{OS Type}|{OS Info}|{Runtime Info}|{App Id}|{Driver Properties} + /// + /// + /// This is the base value, whose {App Id} part is always + /// 0. The payload actually sent at login carries the + /// identifier set on the connection; see + /// GetUcs2Bytes. /// /// /// The {Driver Version} part is the version of the driver, @@ -76,6 +84,17 @@ internal static class UserAgent /// Maximum length is 44 characters. /// /// + /// The {App Id} part is the identifier of the application + /// middleware using the driver, serialized as uppercase hexadecimal. + /// It is always present; 0 means no application identity was + /// reported. Maximum length is 4 characters. + /// + /// + /// The {Driver Properties} part is a driver-owned feature flag + /// value, serialized as uppercase hexadecimal. It is always present. + /// Maximum length is 16 characters. + /// + /// /// Any characters from the sourced values that are not one of the /// following are replaced with underscore ('_'): /// @@ -108,6 +127,62 @@ internal static class UserAgent #region Helpers + /// + /// + /// Returns the UCS-2 encoded payload reporting the given application + /// identifier. + /// + /// + /// When is + /// , is + /// returned, whose App Id part is 0. + /// + /// + /// + /// The application identifier set on the connection being logged in. + /// + /// The UCS-2 encoded payload bytes. + internal static ReadOnlyMemory GetUcs2Bytes(RegisteredApplication app) + { + if (app == RegisteredApplication.Unknown) + { + return Ucs2Bytes; + } + + // Most processes report a single application identifier, so a single + // cached entry serves every login. The pair is cached behind one + // reference so readers never observe a torn ReadOnlyMemory. + AppPayload? cached = Volatile.Read(ref s_appPayload); + if (cached is not null && cached.App == app) + { + return cached.Ucs2Bytes; + } + + ReadOnlyMemory bytes = Encoding.Unicode.GetBytes(BuildPayload(app)); + Volatile.Write(ref s_appPayload, new AppPayload(app, bytes)); + + return bytes; + } + + /// + /// Build the payload string from the current runtime environment, + /// reporting the given application identifier. + /// + /// The application identifier to report. + /// The payload string value. + private static string BuildPayload(RegisteredApplication app) => + Build( + MaxLenOverall, + PayloadVersion, + DriverName, + ThisAssembly.PackageVersion, + RuntimeInformation.ProcessArchitecture, + s_osType, + RuntimeInformation.OSDescription, + RuntimeInformation.FrameworkDescription, + (ushort)app, + (ulong)SqlClientDriverPropertiesResolver.Current); + /// /// Static construction builds the Client Interface Name. All known /// exceptions are consumed. @@ -139,16 +214,11 @@ static UserAgent() } #endif + // Remember it for agent payloads built later. + s_osType = osType; + // Build it! - Value = Build( - MaxLenOverall, - PayloadVersion, - DriverName, - ThisAssembly.PackageVersion, - RuntimeInformation.ProcessArchitecture, - osType, - RuntimeInformation.OSDescription, - RuntimeInformation.FrameworkDescription); + Value = BuildPayload(RegisteredApplication.Unknown); // Convert it to UCS-2 bytes. // @@ -189,6 +259,13 @@ static UserAgent() /// /// The value of the Runtime Info part. /// + /// + /// The value of the App Id part, serialized as uppercase hexadecimal. + /// + /// + /// The value of the Driver Properties part, serialized as uppercase + /// hexadecimal. + /// /// /// The payload string value, never null, never empty, and never longer /// than . @@ -201,7 +278,9 @@ internal static string Build( Architecture arch, string osType, string osInfo, - string runtimeInfo) + string runtimeInfo, + ushort appId, + ulong driverProperties) { string result; @@ -244,6 +323,14 @@ internal static string Build( // Add the Runtime Info, truncating to its max length. name.Append(Truncate(Clean(runtimeInfo), MaxLenRuntimeInfo)); + name.Append('|'); + + // Add the App Id. + name.Append(FormatHex(appId)); + name.Append('|'); + + // Add the Driver Properties. + name.Append(FormatHex(driverProperties)); // Remember the name we've built up. result = name.ToString(); @@ -254,7 +341,8 @@ internal static string Build( // value. result = $"{payloadVersion}|{driverName}|{Unknown}|{Unknown}|" + - $"{Unknown}|{Unknown}|{Unknown}"; + $"{Unknown}|{Unknown}|{Unknown}|{FormatHex(appId)}|" + + $"{FormatHex(driverProperties)}"; } // Truncate to our max length if necessary. @@ -371,6 +459,14 @@ internal static string Clean(string? value) } } + /// + /// Formats the given value as uppercase hexadecimal. + /// + /// The value to format. + /// The formatted value. + internal static string FormatHex(ulong value) => + value.ToString("X", CultureInfo.InvariantCulture); + /// /// Truncate the given value to the given max length, and return the /// result. @@ -398,7 +494,9 @@ internal static string Truncate(string value, ushort maxLength) #region Private Fields // Our payload format version. - private const string PayloadVersion = "1"; + // + // Version 2 adds the App Id and Driver Properties parts. + private const string PayloadVersion = "2"; // Our well-known .NET driver name. private const string DriverName = "MS-MDS"; @@ -428,5 +526,29 @@ internal static string Truncate(string value, ushort maxLength) // unknown, invalid, or when errors occur. private const string Unknown = "Unknown"; + // The OS Type resolved during static construction, retained so payloads + // built later use the same value as Value. + private static readonly string s_osType; + + // The payload for the most recently requested application identifier, + // built on first use. + private static AppPayload? s_appPayload; + + /// + /// Pairs a payload with the application identifier it was built for. + /// + private sealed class AppPayload + { + internal AppPayload(RegisteredApplication app, ReadOnlyMemory ucs2Bytes) + { + App = app; + Ucs2Bytes = ucs2Bytes; + } + + internal RegisteredApplication App { get; } + + internal ReadOnlyMemory Ucs2Bytes { get; } + } + #endregion Private Fields } diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs index c8f18d38bc..27327a1272 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.Designer.cs @@ -484,20 +484,11 @@ internal static string ADP_InvalidMixedUsageOfAccessTokenCallbackAndIntegratedSe } /// - /// Looks up a localized string similar to Cannot set the AccessToken or AccessTokenCallback property if the SspiContextProvider property has been set.. + /// Looks up a localized string similar to Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider.. /// - internal static string ADP_InvalidMixedUsageOfAccessTokenAndSspiContextProvider { + internal static string ADP_InvalidMixedUsageOfAccessTokenProperties { get { - return ResourceManager.GetString("ADP_InvalidMixedUsageOfAccessTokenAndSspiContextProvider", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Cannot set the SspiContextProvider property if the AccessToken or AccessTokenCallback property has been set.. - /// - internal static string ADP_InvalidMixedUsageOfSspiContextProviderAndAccessToken { - get { - return ResourceManager.GetString("ADP_InvalidMixedUsageOfSspiContextProviderAndAccessToken", resourceCulture); + return ResourceManager.GetString("ADP_InvalidMixedUsageOfAccessTokenProperties", resourceCulture); } } diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx index 4e20d68eeb..bd22fd385c 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.cs.resx @@ -2082,11 +2082,8 @@ Pokud je klíčové slovo Integrated Security připojovacího řetězce nastavené na hodnotu true nebo SSPI, nejde nastavit vlastnost AccessTokenCallback. - - Vlastnost AccessToken nebo AccessTokenCallback nelze nastavit, pokud je nastavená vlastnost SspiContextProvider. - - - Vlastnost SspiContextProvider nelze nastavit, pokud je nastavená vlastnost AccessToken nebo AccessTokenCallback. + + Nelze nastavit více než jednu z vlastností AccessToken, AccessTokenCallback nebo SspiContextProvider. Pokud je v připojovacím řetězci nastavená možnost Authentication=Active Directory Default, nejde nastavit vlastnost AccessTokenCallback. @@ -2137,10 +2134,10 @@ Certifikát poskytnutý serverem neodpovídá certifikátu poskytnutému možností ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Soubor certifikátu určený možností 'ServerCertificate' se nepodařilo načíst ani parsovat: '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Byla zadána možnost 'ServerCertificate', ale server nepředložil certifikát, který by s ním bylo možné porovnat. Neplatný pokus o získání třídy JsonDocument ve sloupci {0}. Třída JsonDocument se podporuje jenom pro sloupce typu json. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx index 07aa2c6b3f..5570cfa5f4 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.de.resx @@ -2082,11 +2082,8 @@ AccessTokenCallback-Eigenschaft kann nicht festgelegt werden, wenn das Schlüsselwort für Verbindungszeichenfolgen 'Integrated Security' auf 'true' oder 'SSPI' gesetzt wurde. - - Die Eigenschaft AccessToken oder AccessTokenCallback kann nicht festgelegt werden, wenn die Eigenschaft SspiContextProvider bereits festgelegt wurde. - - - Die Eigenschaft SspiContextProvider kann nicht festgelegt werden, wenn die Eigenschaft AccessToken oder AccessTokenCallback bereits festgelegt wurde. + + Es kann nur eine der Eigenschaften AccessToken, AccessTokenCallback oder SspiContextProvider festgelegt werden. Die Eigenschaft AccessTokenCallback kann nicht festgelegt werden, wenn in der Verbindungszeichenfolge "Authentication=Active Directory Default" angegeben wurde. @@ -2137,10 +2134,10 @@ Das vom Server bereitgestellte Zertifikat stimmt nicht mit dem Zertifikat überein, das über die ServerCertificate-Option bereitgestellt wurde. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Die mit der Option „ServerCertificate“ angegebene Zertifikatdatei konnte nicht geladen oder geparst werden: '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Die Option „ServerCertificate“ wurde angegeben, aber der Server hat kein Zertifikat bereitgestellt, mit dem es verglichen werden kann. Ungültiger Versuch, „JsonDocument“ für Spalte „{0}“ abzurufen. „JsonDocument“ wird nur für Spalten vom Typ JSON unterstützt. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx index 5d912814b4..1f84064549 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.es.resx @@ -2082,11 +2082,8 @@ No se puede establecer la propiedad AccessTokenCallback si la palabra clave de cadena de conexión "Integrated Security" se ha establecido en "true" o "SSPI". - - No se puede establecer la propiedad AccessToken o AccessTokenCallback si ya se ha establecido la propiedad SspiContextProvider. - - - No se puede establecer la propiedad SspiContextProvider si ya se ha establecido la propiedad AccessToken o AccessTokenCallback. + + No se puede establecer más de una de las propiedades AccessToken, AccessTokenCallback o SspiContextProvider. No se puede establecer la propiedad AccessTokenCallback si se ha especificado 'Authentication=Active Directory Default' en la cadena de conexión. @@ -2137,10 +2134,10 @@ El certificado proporcionado por el servidor no coincide con el certificado proporcionado por la opción ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + No se pudo cargar ni analizar el archivo de certificado especificado por la opción "ServerCertificate": "{0}". - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Se especificó la opción 'ServerCertificate', pero el servidor no presentó un certificado que se pudiera comparar con él. Intento no válido de obtener JsonDocument en la columna '{0}'. JsonDocument solo se admite para columnas de tipo json. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx index c0082ed5d8..537c0dd0cf 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.fr.resx @@ -2082,11 +2082,8 @@ Impossible de définir la propriété AccessTokenCallback si le mot clé de chaîne de connexion « Sécurité intégrée » a été défini sur « true » ou « SSPI ». - - Nous ne pouvons pas définir la propriété AccessToken ou AccessTokenCallback si la propriété SspiContextProvider a déjà été définie. - - - Nous ne pouvons pas définir la propriété SspiContextProvider si la propriété AccessToken ou AccessTokenCallback a déjà été définie. + + Nous ne pouvons pas définir plus d’une des propriétés AccessToken, AccessTokenCallback ou SspiContextProvider. Impossible de définir la propriété AccessTokenCallback si 'Authentication=Active Directory Default' a été spécifié dans la chaîne de connexion. @@ -2137,10 +2134,10 @@ Le certificat fourni par le serveur ne correspond pas au certificat fourni par l’option ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Le fichier de certificat spécifié par l’option « ServerCertificate » n’a pas pu être chargé ou analysé : « {0} ». - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + L’option « ServerCertificate » a été spécifiée, mais le serveur n’a présenté aucun certificat qui puisse être comparé à celui-ci. Tentative non valide d’obtention de JsonDocument sur la colonne « {0} ». JsonDocument est uniquement pris en charge pour les colonnes de type JSON. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx index 4708501a33..8310513c43 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.it.resx @@ -2082,11 +2082,8 @@ Non è possibile impostare la proprietà AccessTokenCallback se la parola chiave della stringa di connessione 'Integrated Security' è stata impostata su 'true' o 'SSPI'. - - Non è possibile impostare la proprietà AccessToken o AccessTokenCallback se la proprietà SspiContextProvider è già stata impostata. - - - Non è possibile impostare la proprietà SspiContextProvider se la proprietà AccessToken o AccessTokenCallback è già stata impostata. + + Non è possibile impostare più di una delle proprietà AccessToken, AccessTokenCallback o SspiContextProvider. Impossibile impostare la proprietà AccessTokenCallback se nella stringa di connessione è stato specificato 'Authentication=Active Directory Default'. @@ -2137,10 +2134,10 @@ Il certificato fornito dal server non corrisponde al certificato fornito dall'opzione ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Non è possibile caricare o analizzare il file del certificato specificato dall'opzione 'ServerCertificate: '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + È stata specificata l'opzione 'ServerCertificate', ma il server non ha presentato un certificato confrontabile con essa. Tentativo non valido di ottenere JsonDocument nella colonna '{0}'. JsonDocument è supportato solo per colonne di tipo JSON. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx index e6d556d6d3..f24fe2d297 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ja.resx @@ -2082,11 +2082,8 @@ 'Integrated Security' 接続文字列キーワードが 'true' または 'SSPI' に設定されている場合、AccessTokenCallback プロパティを設定できません。 - - SspiContextProvider プロパティが設定されている場合は、AccessToken または AccessTokenCallback プロパティを設定できません。 - - - AccessToken または AccessTokenCallback プロパティが設定されている場合は、SspiContextProvider プロパティを設定できません。 + + AccessToken、AccessTokenCallback、 SspiContextProvider のプロパティを 2 つ以上設定することはできません。 接続文字列に 'Authentication=Active Directory Default' が指定されている場合、AccessTokenCallback プロパティを設定できません。 @@ -2137,10 +2134,10 @@ サーバーによって提供された証明書が、ServerCertificate オプションで指定された証明書と一致しません。 - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 'ServerCertificate' オプションで指定された証明書ファイルを読み込むことも解析することもできませんでした: '{0}'。 - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 'ServerCertificate' オプションが指定されましたが、サーバーから、このオプションと照合できる証明書が提示されませんでした。 列 '{0}' で JsonDocument を取得しようとしましたが無効です。JsonDocument は json 型の列でのみサポートされています。 diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx index 688a1bee60..ab87cea583 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ko.resx @@ -2082,11 +2082,8 @@ 'Integrated Security' 연결 문자열 키워드가 'true' 또는 'SSPI'로 설정된 경우 AccessTokenCallback 속성을 설정할 수 없습니다. - - SspiContextProvider 속성이 설정된 경우 AccessToken 또는 AccessTokenCallback 속성을 설정할 수 없습니다. - - - AccessToken 또는 AccessTokenCallback 속성이 설정된 경우 SspiContextProvider 속성을 설정할 수 없습니다. + + AccessToken, AccessTokenCallback 또는 SspiContextProvider 속성 중 두 개 이상을 설정할 수 없습니다. 연결 문자열에 'Authentication=Active Directory Default'가 지정된 경우 AccessTokenCallback 속성을 설정할 수 없습니다. @@ -2137,10 +2134,10 @@ 서버에서 제공하는 인증서가 ServerCertificate 옵션에서 제공하는 인증서와 일치하지 않습니다. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 'ServerCertificate' 옵션으로 지정된 인증서 파일을 로드하거나 구문 분석할 수 없습니다. '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 'ServerCertificate' 옵션이 지정되었지만 서버에서 비교할 수 있는 인증서를 제공하지 않았습니다. 열 '{0}'의 JsonDocument를 가져오려는 시도가 잘못되었습니다. JsonDocument는 json 형식의 열에 대해서만 지원됩니다. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx index d9656a1a8c..dcc088af86 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pl.resx @@ -2082,11 +2082,8 @@ Nie można ustawić właściwości AccessTokenCallback, jeśli słowo kluczowe parametrów połączenia „Integrated Security” ma wartość „true” lub „SSPI”. - - Nie można ustawić właściwości AccessToken lub AccessTokenCallback, jeśli ustawiono właściwość SspiContextProvider. - - - Nie można ustawić właściwości SspiContextProvider, jeśli ustawiono właściwość AccessToken lub AccessTokenCallback. + + Nie można ustawić więcej niż jednej właściwości AccessToken, AccessTokenCallback lub SspiContextProvider. Nie można ustawić właściwości AccessTokenCallback, jeśli w parametrach połączenia określono wartość „Authentication=Active Directory Default”. @@ -2137,10 +2134,10 @@ Certyfikat dostarczony przez serwer nie jest zgodny z certyfikatem dostarczonym przez opcję ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Nie można załadować lub przeanalizować pliku certyfikatu określonego przez opcję „ServerCertificate”: „{0}”. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Określono opcję „ServerCertificate”, ale serwer nie przedstawia certyfikatu, który można porównać z nim. Nieprawidłowa próba pobrania pliku JsonDocument w kolumnie „{0}”. Obiekt JsonDocument jest obsługiwany tylko w przypadku kolumn typu JSON. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx index 581068d864..458096ee2a 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.pt-BR.resx @@ -2082,11 +2082,8 @@ Não é possível definir a propriedade AccessTokenCallback se a palavra-chave da cadeia de conexão 'Integrated Security' tiver sido definida como 'true' ou 'SSPI'. - - Não é possível definir a propriedade AccessToken ou AccessTokenCallback se a propriedade SspiContextProvider já tiver sido definida. - - - Não é possível definir a propriedade SspiContextProvider se a propriedade AccessToken ou AccessTokenCallback já tiver sido definida. + + Não é possível definir mais de uma das propriedades AccessToken, AccessTokenCallback ou SspiContextProvider. Não é possível definir a propriedade AccessTokenCallback se 'Authentication=Active Directory Default' tiver sido especificado na cadeia de conexão. @@ -2137,10 +2134,10 @@ O certificado fornecido pelo servidor não corresponde ao certificado fornecido pela opção ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + O arquivo de certificado especificado pela opção ''ServerCertificate'' não pôde ser carregado ou analisado: ''{0}''. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + A opção ''ServerCertificate'' foi especificada, mas o servidor não apresentou um certificado que pudesse ser comparado com ele. Tentativa inválida de obter JsonDocument na coluna ''{0}''. Só há suporte para JsonDocument em colunas do tipo json. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx index 57cbf80016..6e8a93705b 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.resx @@ -2082,11 +2082,8 @@ Cannot set the AccessTokenCallback property if the 'Integrated Security' connection string keyword has been set to 'true' or 'SSPI'. - - Cannot set the AccessToken or AccessTokenCallback property if the SspiContextProvider property has been set. - - - Cannot set the SspiContextProvider property if the AccessToken or AccessTokenCallback property has been set. + + Cannot set more than one of the properties AccessToken, AccessTokenCallback, or SspiContextProvider. Cannot set the AccessTokenCallback property if 'Authentication=Active Directory Default' has been specified in the connection string. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx index 8b3cbff48a..4a14f91101 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.ru.resx @@ -2082,11 +2082,8 @@ Если ключевому слову строки подключения "Integrated Security" задано значение "true" или "SSPI", свойство "AccessTokenCallback" задать не удастся. - - Если задано свойство SspiContextProvider, свойства AccessToken или AccessTokenCallback настроить невозможно. - - - Если заданы свойства AccessToken или AccessTokenCallback, свойство SspiContextProvider настроить невозможно. + + Не удается установить несколько свойств AccessToken, AccessTokenCallback или SspiContextProvider. Если в строке подключения указан параметр "Authentication=Active Directory Default", свойство AccessTokenCallback задать не удастся. @@ -2137,10 +2134,10 @@ Сертификат, предоставленный сервером, не совпадает с сертификатом, указанным в параметре ServerCertificate. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + Не удалось загрузить или проанализировать файл сертификата, указанный параметром "ServerCertificate": "{0}". - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + Параметр "ServerCertificate" был указан, но сервер не представил сертификат, с которым можно было бы его сравнить. Недопустимая попытка получить JsonDocument для столбца "{0}". JsonDocument поддерживается только для столбцов типа JSON. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx index 650a21f087..60698762e5 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.tr.resx @@ -2082,11 +2082,8 @@ 'Integrated Security' bağlantı dizesi anahtar sözcüğü 'true' veya 'SSPI' olarak ayarlanmışsa AccessTokenCallback özelliği ayarlanamaz. - - AccessToken veya AccessTokenCallback özelliği, SspiContextProvider özelliği ayarlanmışsa ayarlanamaz. - - - SspiContextProvider özelliği, AccessToken veya AccessTokenCallback özelliği ayarlanmışsa ayarlanamaz. + + AccessToken, AccessTokenCallback veya SspiContextProvider özelliklerinden birden fazlası ayarlanamaz. Bağlantı dizesinde 'Authentication=Active Directory Default' belirtilmişse AccessTokenCallback özelliği ayarlanamaz. @@ -2137,10 +2134,10 @@ Sunucu tarafından sağlanan sertifika, ServerCertificate seçeneği tarafından sağlanan sertifikayla eşleşmiyor. - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 'ServerCertificate' seçeneğiyle belirtilen sertifika dosyası yüklenemedi veya ayrıştırılamadı: '{0}'. - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 'ServerCertificate' seçeneği belirtildi, ancak sunucu bununla karşılaştırılabilecek bir sertifika sunmadı. '{0}' sütununda JsonDocument alma denemesi geçersiz. JsonDocument yalnızca json türündeki sütunlar için desteklenir. diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx index 77db610a7c..2c6388df6e 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hans.resx @@ -2082,11 +2082,8 @@ 如果 "Integrated Security" 连接字符串关键字设置为 "true" 或 "SSPI",则无法设置 AccessTokenCallback 属性。 - - 如果已设置 SspiContextProvider 属性,则无法设置 AccessToken 或 AccessTokenCallback 属性。 - - - 如果已设置 AccessToken 或 AccessTokenCallback 属性,则无法设置 SspiContextProvider 属性。 + + 只能设置以下属性其中之一: AccessToken、AccessTokenCallback 或 SspiContextProvider。 如果在连接字符串中指定了 "Authentication=Active Directory Default",则无法设置 AccessTokenCallback 属性。 @@ -2137,10 +2134,10 @@ 服务器提供的证书与 ServerCertificate 选项提供的证书不匹配。 - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 无法加载或解析 "ServerCertificate" 选项指定的证书文件: "{0}"。 - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 已指定 "ServerCertificate" 选项,但服务器未提供可与其比较的证书。 在列 ‘{0}’ 上获取 JsonDocument 的尝试无效。只有 json 类型的列支持 JsonDocument。 diff --git a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx index f7d87d6908..d71c496a85 100644 --- a/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx +++ b/src/Microsoft.Data.SqlClient/src/Resources/Strings.zh-Hant.resx @@ -2082,11 +2082,8 @@ 如果 'Integrated Security' 連接字串關鍵字已設定為 'true' 或 'SSPI',就不能設定 AccessTokenCallback 屬性。 - - 如果已設定 SspiContextProvider 屬性,就不能設定 AccessToken 或 AccessTokenCallback 屬性。 - - - 如果已設定 AccessToken 或 AccessTokenCallback 屬性,就不能設定 SspiContextProvider 屬性。 + + AccessToken、AccessTokenCallback 或 SspiContextProvider 這些屬性中,不能設定超過一個。 如果已在連接字串中指定 'Authentication=Active Directory Default',就不能設定 AccessTokenCallback 屬性。 @@ -2137,10 +2134,10 @@ 伺服器提供的憑證不符合 ServerCertificate 選項所提供的憑證。 - The certificate file specified by the 'ServerCertificate' option could not be loaded or parsed: '{0}'. + 無法載入或剖析 'ServerCertificate' 選項所指定的憑證檔案: '{0}'。 - The 'ServerCertificate' option was specified, but the server did not present a certificate that could be compared against it. + 已指定 'ServerCertificate' 選項,但伺服器未提供可與其比對的憑證。 在資料行 '{0}' 上取得 JsonDocument 的嘗試無效。JsonDocument 僅支援 JSON 類型的資料行。 diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/AzureKeyVaultKeyFixtureBase.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/AzureKeyVaultKeyFixtureBase.cs index b2232b3f69..8feb0e38f0 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/AzureKeyVaultKeyFixtureBase.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/AzureKeyVaultKeyFixtureBase.cs @@ -75,18 +75,38 @@ public void Dispose() protected virtual void Dispose(bool disposing) { - foreach (KeyVaultKey key in _createdKeys) + try { - try - { - _keyClient.StartDeleteKey(key.Name).WaitForCompletion(); - } - catch (Exception) + foreach (KeyVaultKey key in _createdKeys) { - continue; + try + { + _keyClient.StartDeleteKey(key.Name).WaitForCompletion(); + } + catch (Exception) + { + continue; + } + + // A deleted key remains in a soft-deleted state (and continues to consume the name) + // until it is purged or the retention period expires. Purging is best-effort: the + // test principal may not have permission, and the vault may have purge protection + // enabled, in which case the soft-deleted key simply expires on its own. + try + { + _keyClient.PurgeDeletedKey(key.Name); + } + catch (Exception) + { + continue; + } } - } - _randomGenerator.Dispose(); + _createdKeys.Clear(); + } + finally + { + _randomGenerator.Dispose(); + } } } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/CertificateFixtureBase.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/CertificateFixtureBase.cs index 5081bc8b18..88b3a2c7b4 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/CertificateFixtureBase.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/CertificateFixtureBase.cs @@ -50,7 +50,23 @@ public CertificateStoreContext(StoreLocation location, StoreName name) private readonly List _certificateStoreModifications = new List(); + /// + /// Every certificate handed out by . Certificates are created with + /// , which writes the private key to a key container on + /// disk. Removing the certificate from a store does not remove that container, so the containers must + /// be deleted explicitly or they accumulate indefinitely across test runs. + /// + private readonly List _createdCertificates = new List(); + protected X509Certificate2 CreateCertificate(string subjectName, IEnumerable dnsNames, IEnumerable ipAddresses, bool forceCsp = false) + { + X509Certificate2 certificate = CreateCertificateCore(subjectName, dnsNames, ipAddresses, forceCsp); + + _createdCertificates.Add(certificate); + return certificate; + } + + private X509Certificate2 CreateCertificateCore(string subjectName, IEnumerable dnsNames, IEnumerable ipAddresses, bool forceCsp = false) { // This will always generate a certificate with: // * Start date: 24hrs ago @@ -281,21 +297,45 @@ public void Dispose() protected virtual void Dispose(bool disposing) { + // Collect everything that needs disposing before touching any of it: removal from a store and + // deletion of the persisted key are both best-effort, but the handles must always be released. + List certificates = new List(_createdCertificates); + + // Remove the certificates from any store they were added to. This must happen before the + // certificates are disposed, because a disposed certificate cannot be matched against a store. foreach (CertificateStoreContext storeContext in _certificateStoreModifications) { using X509Store store = new X509Store(storeContext.Name, storeContext.Location); + bool opened; try { store.Open(OpenFlags.ReadWrite); + opened = true; } - catch (Exception) + catch (Exception ex) { - continue; + // Every certificate destined for this store is about to be skipped, so this is the + // only chance to say so. Without it the certificates leak across runs with no trace. + Console.WriteLine( + $"Failed to open certificate store '{storeContext.Name}' in '{storeContext.Location}' " + + $"for cleanup; {storeContext.Certificates.Count} certificate(s) may be left behind. {ex}"); + + opened = false; } foreach (X509Certificate2 cert in storeContext.Certificates) { + if (!certificates.Contains(cert)) + { + certificates.Add(cert); + } + + if (!opened) + { + continue; + } + try { if (store.Certificates.Contains(cert)) @@ -303,15 +343,75 @@ protected virtual void Dispose(bool disposing) store.Remove(cert); } } - catch (Exception) + catch (Exception ex) { + // Same reasoning as the store-open failure above: report the leak rather than + // letting the certificate linger in the store silently. + Console.WriteLine( + $"Failed to remove certificate '{cert.Subject}' from store '{storeContext.Name}' " + + $"in '{storeContext.Location}'; it may be left behind. {ex}"); + continue; } + } + + storeContext.Certificates.Clear(); + } + + _certificateStoreModifications.Clear(); + foreach (X509Certificate2 cert in certificates) + { + DeletePersistedPrivateKey(cert); + + try + { cert.Dispose(); } + catch (Exception) + { + // Nothing further can be done about a certificate that refuses to release its handle. + } + } - storeContext.Certificates.Clear(); + _createdCertificates.Clear(); + } + + /// + /// Deletes the on-disk key container backing a certificate's private key, if there is one. + /// + /// + /// This is best-effort: the certificate may have no private key, the key may be ephemeral, or the + /// current user may lack permission to delete it (for example, for a machine-scoped key). + /// + private static void DeletePersistedPrivateKey(X509Certificate2 certificate) + { + try + { + if (!certificate.HasPrivateKey) + { + return; + } + + using RSA? privateKey = certificate.GetRSAPrivateKey(); + + switch (privateKey) + { + case RSACryptoServiceProvider csp: + csp.PersistKeyInCsp = false; + break; +#if NET + case RSACng cng: + cng.Key.Delete(); + break; +#endif + default: + break; + } + } + catch (Exception) + { + // Best-effort; a stale key container is preferable to failing the test run during cleanup. } } } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnEncryptionCertificateFixture.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnEncryptionCertificateFixture.cs index c5b527c690..ff2dc0719b 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnEncryptionCertificateFixture.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnEncryptionCertificateFixture.cs @@ -28,29 +28,41 @@ public sealed class ColumnEncryptionCertificateFixture : CertificateFixtureBase public ColumnEncryptionCertificateFixture() { - PrimaryColumnEncryptionCertificate = CreateCertificate(nameof(PrimaryColumnEncryptionCertificate), Array.Empty(), Array.Empty()); - SecondaryColumnEncryptionCertificate = CreateCertificate(nameof(SecondaryColumnEncryptionCertificate), Array.Empty(), Array.Empty()); - _currentUserCertificate = CreateCertificate(nameof(_currentUserCertificate), Array.Empty(), Array.Empty()); - using (X509Certificate2 createdCertificate = CreateCertificate(nameof(CertificateWithoutPrivateKey), Array.Empty(), Array.Empty())) + // NOTE: If this constructor throws, xUnit never calls Dispose, so any certificate already + // placed in a store (and its persisted key container) would be leaked. + try { + PrimaryColumnEncryptionCertificate = CreateCertificate(nameof(PrimaryColumnEncryptionCertificate), Array.Empty(), Array.Empty()); + SecondaryColumnEncryptionCertificate = CreateCertificate(nameof(SecondaryColumnEncryptionCertificate), Array.Empty(), Array.Empty()); + _currentUserCertificate = CreateCertificate(nameof(_currentUserCertificate), Array.Empty(), Array.Empty()); + + // NOTE: The source certificate is intentionally not disposed here; it is tracked by the base + // fixture, which deletes its persisted key container and disposes it on cleanup. + X509Certificate2 createdCertificate = CreateCertificate(nameof(CertificateWithoutPrivateKey), Array.Empty(), Array.Empty()); + // This will strip the private key away from the created certificate #if NET9_0_OR_GREATER - CertificateWithoutPrivateKey = X509CertificateLoader.LoadCertificate(createdCertificate.Export(X509ContentType.Cert)); + CertificateWithoutPrivateKey = X509CertificateLoader.LoadCertificate(createdCertificate.Export(X509ContentType.Cert)); #else CertificateWithoutPrivateKey = new X509Certificate2(createdCertificate.Export(X509ContentType.Cert)); #endif AddToStore(CertificateWithoutPrivateKey, StoreLocation.CurrentUser, StoreName.My); - } - AddToStore(PrimaryColumnEncryptionCertificate, StoreLocation.CurrentUser, StoreName.My); - AddToStore(SecondaryColumnEncryptionCertificate, StoreLocation.CurrentUser, StoreName.My); - AddToStore(_currentUserCertificate, StoreLocation.CurrentUser, StoreName.My); + AddToStore(PrimaryColumnEncryptionCertificate, StoreLocation.CurrentUser, StoreName.My); + AddToStore(SecondaryColumnEncryptionCertificate, StoreLocation.CurrentUser, StoreName.My); + AddToStore(_currentUserCertificate, StoreLocation.CurrentUser, StoreName.My); - if (IsAdmin) - { - _localMachineCertificate = CreateCertificate(nameof(_localMachineCertificate), Array.Empty(), Array.Empty()); + if (IsAdmin) + { + _localMachineCertificate = CreateCertificate(nameof(_localMachineCertificate), Array.Empty(), Array.Empty()); - AddToStore(_localMachineCertificate, StoreLocation.LocalMachine, StoreName.My); + AddToStore(_localMachineCertificate, StoreLocation.LocalMachine, StoreName.My); + } + } + catch + { + Dispose(); + throw; } } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnMasterKeyCertificateFixture.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnMasterKeyCertificateFixture.cs index d396251103..5cb70cc27a 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnMasterKeyCertificateFixture.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/ColumnMasterKeyCertificateFixture.cs @@ -28,11 +28,21 @@ protected ColumnMasterKeyCertificateFixture(bool createCertificate) { if (createCertificate) { - ColumnMasterKeyCertificate = CreateCertificate(nameof(ColumnMasterKeyCertificate), Array.Empty(), Array.Empty()); - - AddToStore(ColumnMasterKeyCertificate, StoreLocation.CurrentUser, StoreName.My); - - ColumnMasterKeyCertificatePath = $"{StoreLocation.CurrentUser}/{StoreName.My}/{ColumnMasterKeyCertificate.Thumbprint}"; + // NOTE: If this constructor throws, xUnit never calls Dispose, so the certificate placed in + // the store (and its persisted key container) would be leaked. + try + { + ColumnMasterKeyCertificate = CreateCertificate(nameof(ColumnMasterKeyCertificate), Array.Empty(), Array.Empty()); + + AddToStore(ColumnMasterKeyCertificate, StoreLocation.CurrentUser, StoreName.My); + + ColumnMasterKeyCertificatePath = $"{StoreLocation.CurrentUser}/{StoreName.My}/{ColumnMasterKeyCertificate.Thumbprint}"; + } + catch + { + Dispose(); + throw; + } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/CspCertificateFixture.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/CspCertificateFixture.cs index 4d46032ce2..ac8e847656 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/CspCertificateFixture.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/CspCertificateFixture.cs @@ -19,12 +19,22 @@ public class CspCertificateFixture : CertificateFixtureBase { public CspCertificateFixture() { - CspCertificate = CreateCertificate(nameof(CspCertificate), Array.Empty(), Array.Empty(), true); + // NOTE: If this constructor throws, xUnit never calls Dispose, so the certificate placed in the + // store (and its persisted CSP key container) would be leaked. + try + { + CspCertificate = CreateCertificate(nameof(CspCertificate), Array.Empty(), Array.Empty(), true); - AddToStore(CspCertificate, StoreLocation.CurrentUser, StoreName.My); + AddToStore(CspCertificate, StoreLocation.CurrentUser, StoreName.My); - CspCertificatePath = $"{StoreLocation.CurrentUser}/{StoreName.My}/{CspCertificate.Thumbprint}"; - CspKeyPath = GetCspPathFromCertificate(); + CspCertificatePath = $"{StoreLocation.CurrentUser}/{StoreName.My}/{CspCertificate.Thumbprint}"; + CspKeyPath = GetCspPathFromCertificate(); + } + catch + { + Dispose(); + throw; + } } public X509Certificate2 CspCertificate { get; } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnEncryptionKey.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnEncryptionKey.cs index af402428b0..bdad3fbdd5 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnEncryptionKey.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnEncryptionKey.cs @@ -27,7 +27,7 @@ public sealed class ColumnEncryptionKey : DatabaseObject /// The column master key which backs this encryption key. public ColumnEncryptionKey(SqlConnection connection, string namePrefix, ColumnMasterKey cmkOrigin) : base(connection, GenerateLongName(namePrefix), definition: DefinitionTemplate, - state: cmkOrigin, shouldCreate: true, shouldDrop: true) + state: cmkOrigin) { } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnMasterKey.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnMasterKey.cs index de607c0f68..0802cbbe53 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnMasterKey.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ColumnMasterKey.cs @@ -37,7 +37,7 @@ internal CreationParameters(SqlColumnEncryptionKeyStoreProvider provider, protected ColumnMasterKey(SqlConnection connection, string namePrefix, CreationParameters creationParameters) : base(connection, name: GenerateLongName(namePrefix), definition: DefinitionTemplate, - state: creationParameters, shouldCreate: true, shouldDrop: true) + state: creationParameters) { } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/DatabaseObject.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/DatabaseObject.cs index a74da79697..a93df61fca 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/DatabaseObject.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/DatabaseObject.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Text; namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; @@ -16,7 +17,7 @@ namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; /// public abstract class DatabaseObject : IDisposable { - private readonly bool _shouldDrop; + private bool _disposed; protected SqlConnection Connection { get; } @@ -26,20 +27,51 @@ public abstract class DatabaseObject : IDisposable public string UnescapedName => Name.Substring(1, Name.Length - 2).Replace("]]", "]"); - protected DatabaseObject(SqlConnection connection, string name, string definition, TState state, bool shouldCreate, bool shouldDrop) + protected DatabaseObject(SqlConnection connection, string name, string definition, TState state) { - _shouldDrop = shouldDrop; - Connection = connection; State = state; Name = name; - if (shouldCreate) + EnsureConnectionOpen(); + + // Remove any object left behind by an earlier run before creating this one. Best-effort: + // if it cannot be removed, CREATE will report the collision far more clearly than a drop + // failure would. + TryDrop(); + + try { - EnsureConnectionOpen(); - DropObject(); CreateObject(definition); } + catch + { + // CREATE can fail *after* the server has created the object: a command timeout or a + // dropped connection reports failure for a statement that may already have committed. + // Every generated name embeds a GUID, so anything left behind is orphaned forever in + // the (shared) test database. Unwind through Dispose so that cleanup lives in exactly + // one place, then let the original failure surface. + Dispose(); + throw; + } + } + + /// + /// Adopts an object that already exists, so that it is dropped when this instance is disposed. + /// Nothing is created. + /// + /// + /// Necessary for objects the server creates as a side effect of creating something else, which + /// therefore have no CREATE statement of their own but still have to be dropped. The temporal + /// history table behind is the motivating case. + /// + protected DatabaseObject(SqlConnection connection, string name, TState state, ExistingObject _) + { + Connection = connection; + State = state; + Name = name; + + EnsureConnectionOpen(); } private void EnsureConnectionOpen() @@ -251,22 +283,127 @@ public static string GenerateShortName(string prefix, bool escape = true) /// /// /// By the time this is called, will be open. - /// Must not throw an exception if the object does not exist. + /// + /// Implementations must be safe to call more than once and must not throw when the object does + /// not exist. They are not required to be exception-free beyond that: the guard against a + /// leftover object is inherently racy (the existence check and the DROP are separate + /// statements), and the connection itself can fail at any moment. owns + /// that problem — it retries on a fresh connection and reports anything it cannot remove — so + /// no failure from here ever escapes to a caller. /// protected abstract void DropObject(); + /// + /// Idempotent, and never throws. These objects are overwhelmingly consumed via using, so + /// a throwing Dispose would surface in place of an exception already in flight and + /// replace a real test failure with a cleanup error — the very "cleanup masks the real failure" + /// problem these types exist to remove. A drop that cannot be completed is reported by + /// instead, which identifies the object and where it lives + /// so the leak stays actionable without destroying the diagnosis of the failure that caused it. + /// public void Dispose() { - if (_shouldDrop) + if (_disposed) { - EnsureConnectionOpen(); - DropObject(); + return; } + + _disposed = true; + + TryDrop(); + // This explicitly does not drop the wrapped SqlConnection; this is sometimes // used in a loop to create multiple UDTs. GC.SuppressFinalize(this); } + + /// + /// Drops the object, swallowing any failure. + /// + /// + /// The drop is all that stands between a failed run and an object orphaned forever in the + /// shared test database, so it gets one retry on a healthy connection before giving up. A + /// failure is never propagated: this runs either during or on a path + /// already unwinding because of a more interesting failure, and in both cases a cleanup error + /// must not replace the exception in flight. + /// + private void TryDrop() + { + try + { + EnsureConnectionOpen(); + DropObject(); + } + catch + { + try + { + TryDropAfterReconnect(); + } + catch + { + // Nothing left to try, and nowhere to report it: even the reporting path failed. + // Swallowing is the whole point — see the remarks above. + } + } + } + + /// + /// Re-attempts the drop on a healthy connection. + /// + /// true if the object was dropped; otherwise false. + /// + /// A drop usually fails because itself has gone bad: a command timeout + /// can leave it unusable, and a transport failure kills it outright. Closing and reopening + /// returns the broken connection to the pool and acquires a healthy one, which is the difference + /// between a transient blip and an object orphaned forever in the shared test database. + /// + /// This deliberately reuses the existing rather than constructing a + /// new one from its connection string: `Persist Security Info` defaults to false, so the password + /// is no longer readable from once it has been + /// opened, and a copy would fail to authenticate. + /// + private bool TryDropAfterReconnect() + { + // Captured before the reconnect attempt: once the connection has been closed or disposed + // these can throw, and this is the only record the leak will leave. + string location = DescribeLocation(); + + try + { + Connection.Close(); + Connection.Open(); + DropObject(); + + return true; + } + catch (Exception ex) + { + // This is the last chance to remove the object, and no caller propagates the failure, + // so this report is the only trace the leak will leave. It names the object *and* + // where it lives, so a human or agent can drop it manually without having to work out + // which server and database the run was pointed at. + Console.WriteLine($"Failed to drop {GetType().Name} '{Name}' {location}; it may be orphaned there. {ex}"); + + return false; + } + } + + /// + /// Describes where the object lives, for the leak report. + /// + private string DescribeLocation() + { + try + { + return $"on data source '{Connection.DataSource}', database '{Connection.Database}'"; + } + catch (Exception ex) + { + return $"on an unknown data source ({ex.GetType().Name} reading the connection)"; + } + } } /// @@ -275,8 +412,13 @@ public void Dispose() /// public abstract class DatabaseObject : DatabaseObject { - protected DatabaseObject(SqlConnection connection, string name, string definition, bool shouldCreate, bool shouldDrop) - : base(connection, name, definition, state: null, shouldCreate, shouldDrop) + protected DatabaseObject(SqlConnection connection, string name, string definition) + : base(connection, name, definition, state: null) + { + } + + protected DatabaseObject(SqlConnection connection, string name, ExistingObject adopt) + : base(connection, name, state: null, adopt) { } } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/DatabaseUser.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/DatabaseUser.cs index a757e9c9e0..20182d1ba3 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/DatabaseUser.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/DatabaseUser.cs @@ -22,7 +22,7 @@ public sealed class DatabaseUser : DatabaseObject /// The name of the database where the user will be created. /// The server login which the database user will be associated with. public DatabaseUser(SqlConnection connection, string database, ServerLogin login) - : base(connection, login.Name, $"FOR LOGIN {login.Name}", state: database, shouldCreate: true, shouldDrop: true) + : base(connection, login.Name, $"FOR LOGIN {login.Name}", state: database) { } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ExistingObject.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ExistingObject.cs new file mode 100644 index 0000000000..93ea622ae8 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ExistingObject.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +/// +/// Selects the constructor that adopts an object which already exists on the server, rather than +/// creating one. +/// +/// +/// The distinction — create this object, or take ownership of one that is already there — cannot be +/// expressed in the signature, since both take the same arguments. A discriminator states it at the +/// call site, which a bool would not; the corresponding public entry point is the +/// AdoptExisting factory. +/// +public enum ExistingObject +{ + Adopt +} diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/NameIsVerbatim.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/NameIsVerbatim.cs new file mode 100644 index 0000000000..1ad15651e7 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/NameIsVerbatim.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +/// +/// Distinguishes a constructor that takes a caller-supplied name verbatim from the prefix-based +/// one, which would otherwise have an identical signature. +/// +/// +/// The two overloads differ only in how the string is interpreted — as a name or as a prefix to +/// generate one from — which the type system cannot express. This makes the private constructor +/// unambiguous without inventing a parameter that carries no meaning; the corresponding public +/// entry point is the WithName factory, where the distinction is stated in the name. +/// +internal enum NameIsVerbatim +{ + Yes +} diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ScalarFunction.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ScalarFunction.cs new file mode 100644 index 0000000000..4044241e76 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ScalarFunction.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +/// +/// A transient scalar user-defined function, created at the start of its scope and dropped when +/// disposed. +/// +public sealed class ScalarFunction : DatabaseObject +{ + /// + /// Initializes a new instance of the ScalarFunction class using the specified SQL connection, + /// name prefix and definition. + /// + /// + /// If a function with the specified name already exists, it will be dropped automatically + /// before creation. + /// + /// The SQL connection used to interact with the database. + /// The prefix for the function name. + /// The SQL definition of the function, following the function name. + public ScalarFunction(SqlConnection connection, string prefix, string definition) + : base(connection, GenerateLongName(prefix), definition) + { + } + + private ScalarFunction(SqlConnection connection, string name, string definition, NameIsVerbatim _) + : base(connection, name, definition) + { + } + + /// + /// Creates a function using the caller-supplied name verbatim, instead of generating one. + /// + /// + /// Prefer the prefix-based constructor: generated names embed a GUID and so cannot collide + /// between concurrent test runs against a shared database. This overload exists for the + /// minority of tests that must control the name exactly, for example because the same function + /// has to be created and addressed over several different connections. + /// + /// The SQL connection used to interact with the database. + /// The function name, already quoted/escaped by the caller if it needs to be. + /// The SQL definition of the function, following the function name. + public static ScalarFunction WithName(SqlConnection connection, string name, string definition) + => new(connection, name, definition, NameIsVerbatim.Yes); + + protected override void CreateObject(string definition) + { + using SqlCommand createCommand = new($"CREATE FUNCTION {Name} {definition}", Connection); + + createCommand.ExecuteNonQuery(); + } + + protected override void DropObject() + { + // NOTE: The name is passed to OBJECT_ID() as a parameter rather than being interpolated + // into a string literal, because it may embed Environment.UserName/MachineName (see + // DatabaseObject.GenerateLongName) and an apostrophe in either would break the batch. + // The identifier in DROP FUNCTION is already bracket-quoted. + using SqlCommand dropCommand = new($"IF (OBJECT_ID(@name) IS NOT NULL) DROP FUNCTION {Name}", Connection); + + dropCommand.Parameters.AddWithValue("@name", Name); + + dropCommand.ExecuteNonQuery(); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/Schema.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/Schema.cs new file mode 100644 index 0000000000..7f316520b5 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/Schema.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +/// +/// A transient schema, created at the start of its scope and dropped when disposed. +/// +/// +/// A schema can only be dropped once it is empty, so any object created inside it must be +/// declared after the schema and therefore disposed before it. +/// +public sealed class Schema : DatabaseObject +{ + /// + /// Initializes a new instance of the Schema class using the specified SQL connection and name prefix. + /// + /// The SQL connection used to interact with the database. + /// The prefix for the schema name. + public Schema(SqlConnection connection, string prefix) + : base(connection, GenerateLongName(prefix), definition: string.Empty) + { + } + + private Schema(SqlConnection connection, string name, NameIsVerbatim _) + : base(connection, name, definition: string.Empty) + { + } + + /// + /// Creates a schema using the caller-supplied name verbatim, instead of generating one. + /// + /// + /// Prefer the prefix-based constructor. This overload exists for tests in which the name + /// itself is under test, for example one containing special characters. + /// + /// The SQL connection used to interact with the database. + /// The schema name, already quoted/escaped by the caller if it needs to be. + public static Schema WithName(SqlConnection connection, string name) + => new(connection, name, NameIsVerbatim.Yes); + + protected override void CreateObject(string definition) + { + // NOTE: CREATE SCHEMA must be the first statement in its batch, so it cannot be guarded + // by an IF the way the other objects are. The base class drops before creating, which + // covers the (vanishingly unlikely) case of a name collision. + using SqlCommand createCommand = new($"CREATE SCHEMA {Name}", Connection); + + createCommand.ExecuteNonQuery(); + } + + protected override void DropObject() + { + // NOTE: The name is passed to SCHEMA_ID() as a parameter rather than being interpolated + // into a string literal, because it may embed Environment.UserName/MachineName (see + // DatabaseObject.GenerateLongName) and an apostrophe in either would break the batch. + // The identifier in DROP SCHEMA is already bracket-quoted. + using SqlCommand dropCommand = new($"IF (SCHEMA_ID(@name) IS NOT NULL) DROP SCHEMA {Name}", Connection); + + dropCommand.Parameters.AddWithValue("@name", UnescapedName); + + dropCommand.ExecuteNonQuery(); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ServerLogin.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ServerLogin.cs index ce264a9d14..721cad1b3b 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ServerLogin.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/ServerLogin.cs @@ -26,7 +26,7 @@ public ServerLogin(SqlConnection connection, string namePrefix, string? defaultD } private ServerLogin(SqlConnection connection, string namePrefix, string password, string? defaultDatabase) - : base(connection, namePrefix, GenerateDefinition(password, defaultDatabase), state: password, shouldCreate: true, shouldDrop: true) + : base(connection, namePrefix, GenerateDefinition(password, defaultDatabase), state: password) { } diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/StoredProcedure.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/StoredProcedure.cs index 0ebbd6cb5e..826c2ccc10 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/StoredProcedure.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/StoredProcedure.cs @@ -21,10 +21,30 @@ public sealed class StoredProcedure : DatabaseObject /// The stored procedure name. Can begin with '#' or '##' to indicate a temporary procedure. /// The SQL definition of the stored procedure. public StoredProcedure(SqlConnection connection, string prefix, string definition) - : base(connection, GenerateLongName(prefix), definition, shouldCreate: true, shouldDrop: true) + : base(connection, GenerateLongName(prefix), definition) { } + private StoredProcedure(SqlConnection connection, string name, string definition, NameIsVerbatim _) + : base(connection, name, definition) + { + } + + /// + /// Creates a stored procedure using the caller-supplied name verbatim, instead of generating one. + /// + /// + /// Prefer the prefix-based constructor: generated names embed a GUID and so cannot collide + /// between concurrent test runs against a shared database. This overload exists for the + /// minority of tests that must control the name exactly, for example because the same + /// procedure has to be created and addressed over several different connections. + /// + /// The SQL connection used to interact with the database. + /// The procedure name, already quoted/escaped by the caller if it needs to be. + /// The SQL definition of the stored procedure. + public static StoredProcedure WithName(SqlConnection connection, string name, string definition) + => new(connection, name, definition, NameIsVerbatim.Yes); + protected override void CreateObject(string definition) { using SqlCommand createCommand = new($"CREATE PROCEDURE {Name} {definition}", Connection); diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/Table.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/Table.cs index 42d07a8fba..b628c3b0fd 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/Table.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/Table.cs @@ -21,10 +21,50 @@ public sealed class Table : DatabaseObject /// The prefix for the table name. Can begin with '#' or '##' to indicate a temporary table. /// The SQL definition describing the structure of the table, including columns and data types. public Table(SqlConnection connection, string prefix, string definition) - : base(connection, GenerateLongName(prefix), definition, shouldCreate: true, shouldDrop: true) + : base(connection, GenerateLongName(prefix), definition) { } + private Table(SqlConnection connection, string name, string definition, NameIsVerbatim _) + : base(connection, name, definition) + { + } + + private Table(SqlConnection connection, string name, ExistingObject adopt) + : base(connection, name, adopt) + { + } + + /// + /// Creates a table using the caller-supplied name verbatim, instead of generating one. + /// + /// + /// Prefer the prefix-based constructor: generated names embed a GUID and so cannot collide + /// between concurrent test runs against a shared database. This overload exists for the + /// minority of tests that must control the name exactly - either because the name itself is + /// under test (for example, one containing special characters), or because the same table has + /// to be addressed through several different connections. + /// + /// The SQL connection used to interact with the database. + /// The table name, already quoted/escaped by the caller if it needs to be. + /// The SQL definition describing the structure of the table, including columns and data types. + public static Table WithName(SqlConnection connection, string name, string definition) + => new(connection, name, definition, NameIsVerbatim.Yes); + + /// + /// Adopts a table that already exists, so that it is dropped when the returned instance is + /// disposed. No table is created. + /// + /// + /// For tables the server creates as a side effect of creating something else, and which + /// therefore have no CREATE statement of their own but must still be dropped — the history + /// table behind a being the motivating case. + /// + /// The SQL connection used to drop the table. + /// The table name, already quoted/escaped by the caller if it needs to be. + public static Table AdoptExisting(SqlConnection connection, string name) + => new(connection, name, ExistingObject.Adopt); + protected override void CreateObject(string definition) { using SqlCommand createCommand = new($"CREATE TABLE {Name} {definition}", Connection); diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/TemporalTable.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/TemporalTable.cs new file mode 100644 index 0000000000..6c225938b8 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/TemporalTable.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; + +/// +/// A transient system-versioned (temporal) table, created at the start of its scope and dropped — +/// together with its history table — when disposed. +/// +/// +/// A temporal table cannot be dropped directly. System versioning has to be switched off first, +/// which severs the link to the history table and leaves that table behind as an ordinary one that +/// must then be dropped in its own right. That history table is created by the server as a side +/// effect, so it is adopted rather than created: this type +/// owns it and drops it, but never issues a CREATE for it. +/// +public sealed class TemporalTable : DatabaseObject +{ + /// + /// The history table backing this temporal table. Dropped along with it. + /// + public Table HistoryTable { get; } + + /// + /// Initializes a new system-versioned table and adopts the history table created alongside it. + /// + /// The SQL connection used to interact with the database. + /// The prefix for the table name. + /// + /// The prefix for the history table name. SYSTEM_VERSIONING requires a schema-qualified history + /// table, so the generated name is qualified with [dbo]. + /// + /// + /// The column definitions, including the PERIOD FOR SYSTEM_TIME clause, in parentheses. The + /// SYSTEM_VERSIONING option naming the history table is appended automatically. + /// + public TemporalTable(SqlConnection connection, string prefix, string historyPrefix, string columns) + : this(connection, GenerateLongName(prefix), $"[dbo].{GenerateLongName(historyPrefix)}", columns, NameIsVerbatim.Yes) + { + } + + private TemporalTable(SqlConnection connection, string name, string historyName, string columns, NameIsVerbatim _) + : base(connection, name, $"{columns} WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = {historyName}))") + { + // Adopted only once the CREATE above has succeeded, since that statement is what brings the + // history table into existence. If it throws, the base constructor disposes this instance, + // and the null check in DropObject keeps that path safe. + HistoryTable = Table.AdoptExisting(connection, historyName); + } + + protected override void CreateObject(string definition) + { + using SqlCommand createCommand = new($"CREATE TABLE {Name} {definition}", Connection); + + createCommand.ExecuteNonQuery(); + } + + protected override void DropObject() + { + // NOTE: The name is passed to OBJECT_ID() as a parameter rather than being interpolated + // into a string literal, because it embeds Environment.UserName/MachineName (see + // DatabaseObject.GenerateLongName) and an apostrophe in either would break the batch. + // The identifier in ALTER/DROP TABLE is already bracket-quoted by GenerateLongName. + // + // Ordering is load-bearing: SYSTEM_VERSIONING must be switched off before the period can + // be dropped, and the period before the table itself. + using (SqlCommand dropCommand = new($""" + IF (OBJECT_ID(@name) IS NOT NULL) + BEGIN + ALTER TABLE {Name} SET (SYSTEM_VERSIONING = OFF); + ALTER TABLE {Name} DROP PERIOD FOR SYSTEM_TIME; + DROP TABLE {Name}; + END + """, Connection)) + { + dropCommand.Parameters.AddWithValue("@name", Name); + + dropCommand.ExecuteNonQuery(); + } + + // Only reachable once versioning is off, which is what turns the history table back into an + // ordinary droppable one. Null both during the pre-emptive drop the base constructor runs + // before CREATE, and while it unwinds a failed CREATE — in neither case is there an adopted + // history table to drop. + HistoryTable?.Dispose(); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/UserDefinedType.cs b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/UserDefinedType.cs index be68e649dd..95c9f8ebf0 100644 --- a/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/UserDefinedType.cs +++ b/src/Microsoft.Data.SqlClient/tests/Common/Fixtures/DatabaseObjects/UserDefinedType.cs @@ -21,7 +21,7 @@ public sealed class UserDefinedType : DatabaseObject /// The type name. /// The SQL definition of the type. public UserDefinedType(SqlConnection connection, string prefix, string definition) - : base(connection, "[dbo]." + GenerateLongName(prefix), definition, shouldCreate: true, shouldDrop: true) + : base(connection, "[dbo]." + GenerateLongName(prefix), definition) { } diff --git a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/LocalizationTest.cs b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/LocalizationTest.cs index 250c009b27..b55f63dc09 100644 --- a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/LocalizationTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/LocalizationTest.cs @@ -40,7 +40,7 @@ public class LocalizationTest [InlineData("ru-RU")] [InlineData("zh-Hans")] [InlineData("zh-Hant")] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public void Localization_Tests(string culture) { string localized = GetLocalizedErrorMessage(culture); diff --git a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConfigurableRetryLogicTest.cs b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConfigurableRetryLogicTest.cs index b04115b191..b24a32e804 100644 --- a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConfigurableRetryLogicTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlConfigurableRetryLogicTest.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System; +using System.IO; +using System.Reflection; using System.Threading.Tasks; using Xunit; @@ -80,5 +82,49 @@ public void ValidateRetryParameters() option.AuthorizedSqlCondition = null; SqlConfigurableRetryFactory.CreateIncrementalRetryProvider(option); } + +#if NET + /// + /// Regression test: triggering the configurable retry logic loader through its normal + /// entry points must not leave a process-wide + /// resolving handler + /// installed. Such a handler participates in resolution of every assembly the host + /// application fails to find, and serves them out of this component's probing directory, + /// which can load code from an unintended location. + /// + [Fact] + public void RetryLogicProviderDoesNotLeaveAssemblyProbingEnabled() + { + // Touch the default retry logic providers to force SqlConfigurableRetryLogicLoader + // construction via its normal code path. + Assert.NotNull(new SqlCommand().RetryLogicProvider); + Assert.NotNull(new SqlConnection().RetryLogicProvider); + + // A file that is not a valid assembly, planted in the loader's probing directory + // under a name no other component could be asking for. If a handler is still + // subscribed it finds this file and fails with BadImageFormatException. With correct + // behavior the runtime never looks here and reports the assembly as simply not found. + string assemblySimpleName = "MdsProbeAssembly_" + Guid.NewGuid().ToString("N"); + string plantedFile = Path.Combine(AppContext.BaseDirectory, assemblySimpleName + ".dll"); + + File.WriteAllText(plantedFile, "not an assembly"); + try + { + Assert.Throws( + () => Assembly.Load(new AssemblyName(assemblySimpleName))); + } + finally + { + try + { + File.Delete(plantedFile); + } + catch (IOException) + { + // Best effort cleanup. + } + } + } +#endif } } diff --git a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlDataRecordTest.cs b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlDataRecordTest.cs index b051f2cc26..c39f733a86 100644 --- a/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlDataRecordTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlDataRecordTest.cs @@ -334,7 +334,7 @@ public void GetChar_ThrowsNotSupported() [Theory] #if NETFRAMEWORK - [Trait("Category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server + [Trait("category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server #endif [MemberData( nameof(GetUdtTypeTestData.Get), diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs index 8ecb3c840c..49a6eecc55 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs @@ -1999,7 +1999,7 @@ public void TestBeginAndEndExecuteReaderWithAsyncCallback(string connection, Com } } - [Trait("Category", "flaky")] // System.AggregateException : One or more errors occurred. (Assert.ThrowsAny() Failure: No exception was thrown + [Trait("category", "flaky")] // System.AggregateException : One or more errors occurred. (Assert.ThrowsAny() Failure: No exception was thrown // Expected: typeof(System.Exception)) [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTargetReadyForAeWithKeyStore))] [ClassData(typeof(AEConnectionStringProviderWithExecutionMethod))] @@ -2232,7 +2232,7 @@ public void TestSqlCommandCancel(string connection, string value) // fixed CancelAfter window races a real, network-latency-dependent Always Encrypted // query and the exact-message assertion flakes. Quarantine it only for non-DEBUG // builds; under DEBUG the failpoint keeps it deterministic and gating. - [Trait("Category", "flaky")] + [Trait("category", "flaky")] #endif [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTargetReadyForAeWithKeyStore))] [ClassData(typeof(AEConnectionStringProviderWithCancellationTime))] diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/EnclaveAzureDatabaseTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/EnclaveAzureDatabaseTests.cs index 4de82d39b7..7fca0a7abb 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/EnclaveAzureDatabaseTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/EnclaveAzureDatabaseTests.cs @@ -50,14 +50,25 @@ public EnclaveAzureDatabaseTests(AzureKeyVaultKeyFixture keyVaultKeyFixture) connStrings.Add(connString1.ToString()); connStrings.Add(connString2.ToString()); - foreach (string connString in connStrings) + // NOTE: If creation fails part way through (for example on the second database), this + // constructor never returns, so xUnit never calls Dispose and the keys created so far + // would be leaked. Their names embed a GUID, so nothing would ever reclaim them. + try { - using (SqlConnection connection = new SqlConnection(connString)) + foreach (string connString in connStrings) { - connection.Open(); - databaseObjects.ForEach(o => o.Create(connection)); + using (SqlConnection connection = new SqlConnection(connString)) + { + connection.Open(); + databaseObjects.ForEach(o => o.Create(connection)); + } } } + catch + { + Dispose(); + throw; + } } } @@ -162,13 +173,36 @@ public void Dispose() { if (DataTestUtility.IsEnclaveAzureDatabaseSetup()) { - databaseObjects.Reverse(); + // NOTE: Reverse a copy - reversing the field in place would restore creation order if + // this ever ran twice, dropping the master key before the encryption key that depends + // on it. Every drop is best-effort so that one failure cannot leak the rest. + List objectsToDrop = new List(databaseObjects); + objectsToDrop.Reverse(); + foreach (string connStr in connStrings) { - using (SqlConnection sqlConnection = new SqlConnection(connStr)) + try + { + using (SqlConnection sqlConnection = new SqlConnection(connStr)) + { + sqlConnection.Open(); + + foreach (DbObject databaseObject in objectsToDrop) + { + try + { + databaseObject.Drop(sqlConnection); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(EnclaveAzureDatabaseTests)}: failed to drop '{databaseObject.Name}': {ex.Message}"); + } + } + } + } + catch (Exception ex) { - sqlConnection.Open(); - databaseObjects.ForEach(o => o.Drop(sqlConnection)); + Console.WriteLine($"{nameof(EnclaveAzureDatabaseTests)}: cleanup connection failed: {ex.Message}"); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ExceptionsGenericError.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ExceptionsGenericError.cs index 00e45951db..a7cac6b476 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ExceptionsGenericError.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ExceptionsGenericError.cs @@ -3,7 +3,9 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Data; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted @@ -116,64 +118,100 @@ public sealed class ExceptionGenericErrorFixture : IDisposable static public string encryptedTableName; static public string encryptedProcedureName; + private readonly List _databaseObjects = new(); + private readonly List _connections = new(); + public ExceptionGenericErrorFixture() { SqlConnection.ColumnEncryptionQueryMetadataCacheEnabled = false; - CreateAndPopulateSimpleTable(); + + // NOTE: If setup fails part way through, this constructor never returns, so xUnit never + // calls Dispose and the objects created so far would be leaked. Their names embed a + // GUID, so anything left behind stays in the shared test database forever. + try + { + CreateAndPopulateSimpleTable(); + } + catch + { + Dispose(); + throw; + } } private void CreateAndPopulateSimpleTable() { encryptedTableName = DatabaseHelper.GenerateUniqueName("encrypted"); encryptedProcedureName = DatabaseHelper.GenerateUniqueName("encrypted"); + + // The same table and procedure name has to exist behind every AE connection string, so + // the objects are created with an explicit name rather than a generated one, and the + // connection each was created on is held open for the lifetime of the fixture. foreach (string connectionStr in DataTestUtility.AEConnStringsSetup) { - using (SqlConnection conn = CertificateUtility.GetOpenConnection(false, new SqlConnectionStringBuilder(connectionStr))) + SqlConnection conn = CertificateUtility.GetOpenConnection(false, new SqlConnectionStringBuilder(connectionStr)); + _connections.Add(conn); + + _databaseObjects.Add(Table.WithName(conn, encryptedTableName, "(c1 int)")); + + using (SqlCommand cmdInsert = new SqlCommand($"insert into {encryptedTableName} values(1)", conn)) { - using (SqlCommand cmdCreate = new SqlCommand($"create table {encryptedTableName}(c1 int)", conn)) - { - cmdCreate.CommandType = CommandType.Text; - cmdCreate.ExecuteNonQuery(); - } - using (SqlCommand cmdInsert = new SqlCommand($"insert into {encryptedTableName} values(1)", conn)) - { - cmdInsert.CommandType = CommandType.Text; - cmdInsert.ExecuteNonQuery(); - } - using (SqlCommand cmdCreateProc = new SqlCommand($"create procedure {encryptedProcedureName}(@c1 int) as insert into {encryptedTableName} values (@c1)", conn)) - { - cmdCreateProc.CommandType = CommandType.Text; - cmdCreateProc.ExecuteNonQuery(); - } + cmdInsert.CommandType = CommandType.Text; + cmdInsert.ExecuteNonQuery(); } + + _databaseObjects.Add(StoredProcedure.WithName( + conn, encryptedProcedureName, $"(@c1 int) as insert into {encryptedTableName} values (@c1)")); } } public void Dispose() { // Do NOT remove certificate for concurrent consistency. Certificates are used for other test cases as well. - foreach (string connectionStr in DataTestUtility.AEConnStringsSetup) + + // Disposed in reverse creation order so that each procedure is dropped before the table + // it writes into. + for (int i = _databaseObjects.Count - 1; i >= 0; i--) { - SqlConnectionStringBuilder sb = new SqlConnectionStringBuilder(connectionStr); - using (SqlConnection conn = CertificateUtility.GetOpenConnection(false, sb)) + DisposeSafely(_databaseObjects[i]); + } + _databaseObjects.Clear(); + + foreach (SqlConnection conn in _connections) + { + DisposeSafely(conn); + } + _connections.Clear(); + + // Only use traceoff for non-sysadmin role accounts, Azure accounts does not have the permission. + if (DataTestUtility.IsNotAzureServer()) + { + foreach (string connectionStr in DataTestUtility.AEConnStringsSetup) { - using (SqlCommand cmd = new SqlCommand($"drop table {encryptedTableName}", conn)) + try { - cmd.CommandType = CommandType.Text; - cmd.ExecuteNonQuery(); - - cmd.CommandText = $"drop procedure {encryptedProcedureName}"; - cmd.ExecuteNonQuery(); + CertificateUtility.ChangeServerTceSetting(true, new SqlConnectionStringBuilder(connectionStr)); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(ExceptionGenericErrorFixture)}: failed to reset TCE setting: {ex.Message}"); } } + } + } - // Only use traceoff for non-sysadmin role accounts, Azure accounts does not have the permission. - if (DataTestUtility.IsNotAzureServer()) - { - CertificateUtility.ChangeServerTceSetting(true, sb); - } + private static void DisposeSafely(IDisposable disposable) + { + try + { + disposable.Dispose(); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(ExceptionGenericErrorFixture)}: cleanup failed: {ex.Message}"); } } + } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/SqlNullValues.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/SqlNullValues.cs index f409a7db77..0ae70c5aca 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/SqlNullValues.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/SqlNullValues.cs @@ -7,6 +7,9 @@ using System.Collections.Generic; using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted.Setup; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; +// NOTE: Aliased because the AE test fixtures declare their own Table type alongside the shared one. +using SetupTable = Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted.Setup.Table; using Xunit; namespace Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted @@ -16,6 +19,8 @@ public sealed class SqlNullValuesTests : IClassFixture _databaseObjects = new(); + private readonly List _connections = new(); private string UdfName = DatabaseHelper.GenerateUniqueName("SqlNullValuesRetVal"); private string UdfNameNotNull = DatabaseHelper.GenerateUniqueName("SqlNullValuesRetValNotNull"); @@ -26,35 +31,47 @@ public SqlNullValuesTests(SQLSetupStrategyCertStoreProvider context) // Disable the cache to avoid false failures. SqlConnection.ColumnEncryptionQueryMetadataCacheEnabled = false; - foreach (string connStr in DataTestUtility.AEConnStringsSetup) + // NOTE: If setup fails part way through, this constructor never returns, so xUnit never calls + // Dispose and the functions created so far (whose names embed a GUID) would be leaked. + try { - // Insert data and create functions for SqlNullValues test. - using (SqlConnection sqlConnection = new SqlConnection(connStr)) + foreach (string connStr in DataTestUtility.AEConnStringsSetup) { - sqlConnection.Open(); - - using (SqlCommand cmd = new SqlCommand(string.Format("INSERT INTO [{0}] (c1) VALUES (@c1)", tableName), sqlConnection, null, SqlCommandColumnEncryptionSetting.Enabled)) + // Insert data and create functions for SqlNullValues test. + using (SqlConnection sqlConnection = new SqlConnection(connStr)) { - SqlParameter param = cmd.Parameters.Add("@c1", SqlDbType.Int); - param.Value = DBNull.Value; - cmd.ExecuteNonQuery(); + sqlConnection.Open(); - param.Value = 10; - cmd.ExecuteNonQuery(); - } + using (SqlCommand cmd = new SqlCommand(string.Format("INSERT INTO [{0}] (c1) VALUES (@c1)", tableName), sqlConnection, null, SqlCommandColumnEncryptionSetting.Enabled)) + { + SqlParameter param = cmd.Parameters.Add("@c1", SqlDbType.Int); + param.Value = DBNull.Value; + cmd.ExecuteNonQuery(); - string sql1 = $"CREATE FUNCTION {UdfName}() RETURNS INT AS \n BEGIN \n RETURN (SELECT c1 FROM [{tableName}] WHERE c1 IS NULL)\n END"; - string sql2 = $"CREATE FUNCTION {UdfNameNotNull}() RETURNS INT AS \n BEGIN \n RETURN (SELECT c1 FROM [{tableName}] WHERE c1 IS NOT NULL)\n END"; - using (SqlCommand cmd = sqlConnection.CreateCommand()) - { - cmd.CommandText = sql1; - cmd.ExecuteNonQuery(); + param.Value = 10; + cmd.ExecuteNonQuery(); + } - cmd.CommandText = sql2; - cmd.ExecuteNonQuery(); } + + // The same function names have to exist behind every AE connection string, so + // they are created with an explicit name rather than a generated one, and the + // connection each was created on is held open until cleanup. + SqlConnection functionConnection = new SqlConnection(connStr); + functionConnection.Open(); + _connections.Add(functionConnection); + + _databaseObjects.Add(ScalarFunction.WithName(functionConnection, UdfName, + $"() RETURNS INT AS \n BEGIN \n RETURN (SELECT c1 FROM [{tableName}] WHERE c1 IS NULL)\n END")); + _databaseObjects.Add(ScalarFunction.WithName(functionConnection, UdfNameNotNull, + $"() RETURNS INT AS \n BEGIN \n RETURN (SELECT c1 FROM [{tableName}] WHERE c1 IS NOT NULL)\n END")); } } + catch + { + Dispose(); + throw; + } } [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTargetReadyForAeWithKeyStore))] @@ -156,15 +173,45 @@ public void NullValueTests(string connString, ConnStringColumnEncryptionSetting public void Dispose() { + for (int i = _databaseObjects.Count - 1; i >= 0; i--) + { + TryCleanup(_databaseObjects[i].Dispose); + } + _databaseObjects.Clear(); + + foreach (SqlConnection connection in _connections) + { + TryCleanup(connection.Dispose); + } + _connections.Clear(); + foreach (string connStrAE in DataTestUtility.AEConnStringsSetup) { - using (SqlConnection sqlConnection = new SqlConnection(connStrAE)) + try { - sqlConnection.Open(); - Table.DeleteData(fixture.SqlNullValuesTable.Name, sqlConnection); - DataTestUtility.DropFunction(sqlConnection, UdfName); - DataTestUtility.DropFunction(sqlConnection, UdfNameNotNull); + using (SqlConnection sqlConnection = new SqlConnection(connStrAE)) + { + sqlConnection.Open(); + + TryCleanup(() => SetupTable.DeleteData(fixture.SqlNullValuesTable.Name, sqlConnection)); + } } + catch (Exception ex) + { + Console.WriteLine($"{nameof(SqlNullValuesTests)}: cleanup connection failed: {ex.Message}"); + } + } + } + + private static void TryCleanup(Action cleanupAction) + { + try + { + cleanupAction(); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(SqlNullValuesTests)}: cleanup step failed: {ex.Message}"); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/ConversionTestFixture.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/ConversionTestFixture.cs index abae589981..7b34a6314e 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/ConversionTestFixture.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/ConversionTestFixture.cs @@ -46,16 +46,38 @@ public ConversionTestFixture() _columnMasterKey, _certStoreProvider); - foreach (string connectionStr in DataTestUtility.AEConnStringsSetup) + // NOTE: If creation fails part way through (for example on the second connection string), + // this constructor never returns, so xUnit never disposes the fixture and the keys + // created so far would be leaked into the shared test database forever. Dispose + // explicitly before rethrowing; the drops are guarded, so dropping a key that was + // never created is a no-op. + try { - SqlConnectionStringBuilder connectionString = new SqlConnectionStringBuilder(connectionStr); - // The AE setup often fails with a connect timeout here; ensure a reasonable minimum. - connectionString.ConnectTimeout = Math.Max(connectionString.ConnectTimeout, 30); + foreach (string connectionStr in DataTestUtility.AEConnStringsSetup) + { + SqlConnectionStringBuilder connectionString = new SqlConnectionStringBuilder(connectionStr); + // The AE setup often fails with a connect timeout here; ensure a reasonable minimum. + connectionString.ConnectTimeout = Math.Max(connectionString.ConnectTimeout, 30); + + using SqlConnection sqlConnection = new SqlConnection(connectionString.ConnectionString); + sqlConnection.Open(); + _columnMasterKey.Create(sqlConnection); + ColumnEncryptionKey.Create(sqlConnection); + } + } + catch + { + try + { + Dispose(); + } + catch (Exception disposeException) + { + Console.WriteLine( + $"ConversionTestFixture: cleanup after failed setup did not complete: {disposeException.Message}"); + } - using SqlConnection sqlConnection = new SqlConnection(connectionString.ConnectionString); - sqlConnection.Open(); - _columnMasterKey.Create(sqlConnection); - ColumnEncryptionKey.Create(sqlConnection); + throw; } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategy.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategy.cs index 9929db55d8..5cf8a7b22b 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategy.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategy.cs @@ -69,6 +69,38 @@ protected SQLSetupStrategy(string customKeyPath) ColumnMasterKeyPath = customKeyPath; } + /// + /// Runs the supplied setup action, disposing this fixture if the action throws. + /// + /// + /// Derived types perform their setup from their constructors. When a constructor throws, + /// xUnit never receives the instance and therefore never disposes it, so every column master + /// key, column encryption key and table created before the failure would be leaked into the + /// (shared, long lived) test database. Because the object names embed a GUID, those leaks are + /// permanent and accumulate on every run, eventually exhausting SQL Server's per-database + /// object identifiers (error 3807). + /// + protected void SetupOrCleanUp(Action setupAction) + { + try + { + setupAction(); + } + catch + { + try + { + Dispose(); + } + catch (Exception disposeException) + { + Console.WriteLine($"{GetType().Name}: cleanup after failed setup did not complete: {disposeException}"); + } + + throw; + } + } + internal virtual void SetupDatabase() { foreach (string value in DataTestUtility.AEConnStringsSetup) @@ -274,16 +306,48 @@ protected List CreateTables(IList columnEncryptionKe protected override void Dispose(bool disposing) { - databaseObjects.Reverse(); - foreach (string value in DataTestUtility.AEConnStringsSetup) + try { - using (SqlConnection sqlConnection = new SqlConnection(value)) + foreach (string value in DataTestUtility.AEConnStringsSetup) { - sqlConnection.Open(); - databaseObjects.ForEach(o => o.Drop(sqlConnection)); + try + { + using (SqlConnection sqlConnection = new SqlConnection(value)) + { + sqlConnection.Open(); + + // NOTE: Objects are dropped in reverse creation order so that dependants + // (tables) are removed before their dependencies (keys). A local copy is + // reversed rather than the field itself, so that repeated disposal (which + // can happen when setup fails) does not restore the original order. + List objectsToDrop = new List(databaseObjects); + objectsToDrop.Reverse(); + + foreach (DbObject databaseObject in objectsToDrop) + { + // Each drop is best-effort: one failure must not prevent the remaining + // objects from being dropped, or the certificate from being removed. + try + { + databaseObject.Drop(sqlConnection); + } + catch (Exception ex) + { + Console.WriteLine($"{GetType().Name}: failed to drop '{databaseObject.Name}': {ex.Message}"); + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($"{GetType().Name}: failed to clean up database objects: {ex.Message}"); + } } } - base.Dispose(disposing); + finally + { + base.Dispose(disposing); + } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyAzureKeyVault.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyAzureKeyVault.cs index f1ebc3a93e..eac615af55 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyAzureKeyVault.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyAzureKeyVault.cs @@ -35,8 +35,12 @@ public SQLSetupStrategyAzureKeyVault() : base() { RegisterGlobalProviders(AkvStoreProvider); } - SetupAzureKeyVault(); - SetupDatabase(); + + SetupOrCleanUp(() => + { + SetupAzureKeyVault(); + SetupDatabase(); + }); } public static void RegisterGlobalProviders(SqlColumnEncryptionAzureKeyVaultProvider akvProvider) @@ -109,18 +113,39 @@ internal override void SetupDatabase() protected override void Dispose(bool disposing) { - base.Dispose(disposing); - - foreach (string keyName in _akvKeyNames) + try { - try - { - _keyClient.StartDeleteKey(keyName); - } - catch (Exception) + base.Dispose(disposing); + } + finally + { + foreach (string keyName in _akvKeyNames) { - continue; + try + { + // NOTE: The deletion is awaited so that the key is actually removed before the + // process exits; StartDeleteKey only begins the (long running) operation. + _keyClient.StartDeleteKey(keyName).WaitForCompletion(); + + // A deleted key stays soft-deleted (still consuming its name and vault quota) + // until purged. Purging is best-effort: the principal may lack permission, or + // the vault may have purge protection enabled. + try + { + _keyClient.PurgeDeletedKey(keyName); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(SQLSetupStrategyAzureKeyVault)}: failed to purge AKV key '{keyName}': {ex.Message}"); + } + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(SQLSetupStrategyAzureKeyVault)}: failed to delete AKV key '{keyName}': {ex.Message}"); + } } + + _akvKeyNames.Clear(); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyCertStoreProvider.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyCertStoreProvider.cs index db935a5fed..eb2b6c3e32 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyCertStoreProvider.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SQLSetupStrategyCertStoreProvider.cs @@ -16,7 +16,7 @@ public class SQLSetupStrategyCertStoreProvider : SQLSetupStrategy public SQLSetupStrategyCertStoreProvider() : base() { CertStoreProvider = new SqlColumnEncryptionCertificateStoreProvider(); - SetupDatabase(); + SetupOrCleanUp(SetupDatabase); } internal override void SetupDatabase() diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/ColumnEncryptionKey.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/ColumnEncryptionKey.cs index f9b008ddbf..90379e7bbf 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/ColumnEncryptionKey.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/ColumnEncryptionKey.cs @@ -43,12 +43,20 @@ WITH VALUES ( public override void Drop(SqlConnection sqlConnection) { - string sql = $"DROP COLUMN ENCRYPTION KEY [{Name}]"; + // NOTE: The drop is guarded so that cleanup is idempotent. An unguarded DROP throws when + // the key was never created (for example when setup failed part way through), which + // would abort the enclosing drop loop and leak every remaining object. + // NOTE: T-SQL cannot parameterize an identifier, so the name is parameterized in the + // guard - where it is compared as a string - but must be interpolated into the DROP + // itself. The interpolated identifier is bracket-quoted, and the value only ever + // comes from the test's own generated name. + string sql = $"IF EXISTS (SELECT 1 FROM sys.column_encryption_keys WHERE name = @name) DROP COLUMN ENCRYPTION KEY [{Name}]"; using (SqlCommand command = sqlConnection.CreateCommand()) { command.CommandText = sql; command.CommandTimeout = 60; + command.Parameters.AddWithValue("@name", Name); command.ExecuteNonQuery(); } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/ColumnMasterKey.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/ColumnMasterKey.cs index 2e8d67b95c..5de2bf4d1f 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/ColumnMasterKey.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/ColumnMasterKey.cs @@ -50,12 +50,20 @@ public override void Create(SqlConnection sqlConnection) public override void Drop(SqlConnection sqlConnection) { - string sql = $"DROP COLUMN MASTER KEY [{Name}];"; + // NOTE: The drop is guarded so that cleanup is idempotent. An unguarded DROP throws when + // the key was never created (for example when setup failed part way through), which + // would abort the enclosing drop loop and leak every remaining object. + // NOTE: T-SQL cannot parameterize an identifier, so the name is parameterized in the + // guard - where it is compared as a string - but must be interpolated into the DROP + // itself. The interpolated identifier is bracket-quoted, and the value only ever + // comes from the test's own generated name. + string sql = $"IF EXISTS (SELECT 1 FROM sys.column_master_keys WHERE name = @name) DROP COLUMN MASTER KEY [{Name}];"; using (SqlCommand command = sqlConnection.CreateCommand()) { command.CommandText = sql; command.CommandTimeout = 60; + command.Parameters.AddWithValue("@name", Name); command.ExecuteNonQuery(); } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/Table.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/Table.cs index b18e5a0ae8..70def1b1d4 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/Table.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/Setup/Table.cs @@ -12,11 +12,23 @@ protected Table(string name) : base(name) public override void Drop(SqlConnection sqlConnection) { - string sql = $"DROP TABLE [{Name}];"; + // NOTE: The drop is guarded so that cleanup is idempotent. An unguarded DROP throws when + // the object was never created (for example when setup failed part way through), which + // would abort the enclosing drop loop and leak every remaining object. + // NOTE: Both the lookup and the DROP are schema-qualified to [dbo] to match the CREATE + // TABLE statements in the derived classes. An unqualified name resolves against the + // connection's default schema, so if that is not dbo the guard would return NULL and + // silently skip the drop, leaking the table. + // NOTE: T-SQL cannot parameterize an identifier, so the name is parameterized in the + // guard - where it is compared as a string - but must be interpolated into the DROP + // itself. The interpolated identifier is bracket-quoted, and the value only ever + // comes from the test's own generated name. + string sql = $"IF (OBJECT_ID(@name) IS NOT NULL) DROP TABLE [dbo].[{Name}];"; using (SqlCommand command = sqlConnection.CreateCommand()) { command.CommandText = sql; + command.Parameters.AddWithValue("@name", $"[dbo].[{Name}]"); command.ExecuteNonQuery(); } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SqlSetupStrategyCspProvider.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SqlSetupStrategyCspProvider.cs index 08cdfe48ec..da6b06d5d8 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SqlSetupStrategyCspProvider.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/TestFixtures/SqlSetupStrategyCspProvider.cs @@ -27,7 +27,7 @@ public SQLSetupStrategyCspProvider(CspParameters cspParameters) _cspKeyParameters.Add(cspParameters); CspProvider = new SqlColumnEncryptionCspProvider(); - SetupDatabase(); + SetupOrCleanUp(SetupDatabase); } public SqlColumnEncryptionCspProvider CspProvider { get; } @@ -50,28 +50,35 @@ internal override void SetupDatabase() protected override void Dispose(bool disposing) { - foreach (CspParameters cspParameters in _cspKeyParameters) + try { - try + foreach (CspParameters cspParameters in _cspKeyParameters) { - // Create a new instance of RSACryptoServiceProvider. - // Pass the CspParameters class to use the - // key in the container. - using RSACryptoServiceProvider rsaAlg = new RSACryptoServiceProvider(cspParameters); - - // Delete the key entry in the container. - rsaAlg.PersistKeyInCsp = false; - - // Call Clear to release resources and delete the key from the container. - rsaAlg.Clear(); - } - catch (Exception) - { - continue; + try + { + // Create a new instance of RSACryptoServiceProvider. + // Pass the CspParameters class to use the + // key in the container. + using RSACryptoServiceProvider rsaAlg = new RSACryptoServiceProvider(cspParameters); + + // Delete the key entry in the container. + rsaAlg.PersistKeyInCsp = false; + + // Call Clear to release resources and delete the key from the container. + rsaAlg.Clear(); + } + catch (Exception) + { + continue; + } } - } - base.Dispose(disposing); + _cspKeyParameters.Clear(); + } + finally + { + base.Dispose(disposing); + } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug84548.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug84548.cs index fad361ff9e..5c8257999b 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug84548.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug84548.cs @@ -4,6 +4,7 @@ using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,73 +17,57 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string targettable = DataTestUtility.GetShortName("SqlBulkCopyTest_Bug84548", false); - string targetCustomerTable = targettable + "_customer"; - using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) - { - dstConn.Open(); - try - { - Helpers.TryExecute(dstCmd, "CREATE TABLE [" + targetCustomerTable + "] ([CustomerID] [nchar] (5) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, CONSTRAINT [PK_" + targetCustomerTable + "] PRIMARY KEY CLUSTERED (CustomerID) ON [PRIMARY]) ON [PRIMARY]"); + using SqlConnection dstConn = new SqlConnection(dstConstr); + dstConn.Open(); - Helpers.TryExecute(dstCmd, - "CREATE TABLE [" + targettable + "] ([OrderID] [int] NOT NULL , " + - " [CustomerID] [nchar] (5) COLLATE SQL_Latin1_General_CP1_CI_AS NULL , " + - " CONSTRAINT [PK_" + targettable + "] PRIMARY KEY CLUSTERED " + - " (" + - " [OrderID]" + - " ) ON [PRIMARY] ," + - " CONSTRAINT [FK_" + targettable + "_Customers] FOREIGN KEY " + - " (" + - " [CustomerID]" + - " ) REFERENCES [" + targetCustomerTable + "] (" + - " [CustomerID]" + - " )" + - ") ON [PRIMARY]"); + // The order table takes a foreign key on the customer table, so it must be dropped first. + // Disposal runs in reverse declaration order, which gives that for free. + using Table customerTable = new Table(dstConn, "SqlBulkCopyTest_Bug84548_customer", + "([CustomerID] [nchar] (5) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL," + + " PRIMARY KEY CLUSTERED (CustomerID) ON [PRIMARY]) ON [PRIMARY]"); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - { - srcConn.Open(); + using Table orderTable = new Table(dstConn, "SqlBulkCopyTest_Bug84548", + "([OrderID] [int] NOT NULL," + + " [CustomerID] [nchar] (5) COLLATE SQL_Latin1_General_CP1_CI_AS NULL," + + " PRIMARY KEY CLUSTERED ([OrderID]) ON [PRIMARY]," + + $" FOREIGN KEY ([CustomerID]) REFERENCES {customerTable.Name} ([CustomerID])" + + ") ON [PRIMARY]"); - // First copy the customer ID list across - SqlCommand customerCommand = new SqlCommand("SELECT CustomerID from Customers", srcConn); - using (DbDataReader reader = customerCommand.ExecuteReader()) - { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = targetCustomerTable; - bulkcopy.WriteToServer(reader); - } - } + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + { + srcConn.Open(); - SqlCommand srcCmd = new SqlCommand("select OrderID, CustomerID from Orders where OrderId = 10643", srcConn); - using (DbDataReader reader = srcCmd.ExecuteReader()) - { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = targettable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + // First copy the customer ID list across + using SqlCommand customerCommand = new SqlCommand("SELECT CustomerID from Customers", srcConn); + using (DbDataReader reader = customerCommand.ExecuteReader()) + { + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = customerTable.Name; + bulkcopy.WriteToServer(reader); + } + } - ColumnMappings.Add("OrderID", "OrderID"); - ColumnMappings.Add("CustomerID", "CustomerID"); + using SqlCommand srcCmd = new SqlCommand("select OrderID, CustomerID from Orders where OrderId = 10643", srcConn); + using (DbDataReader reader = srcCmd.ExecuteReader()) + { + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = orderTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - bulkcopy.WriteToServer(reader); + ColumnMappings.Add("OrderID", "OrderID"); + ColumnMappings.Add("CustomerID", "CustomerID"); - DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 1, "Unexpected number of rows."); - } - } + bulkcopy.WriteToServer(reader); + + DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 1, "Unexpected number of rows."); } - Helpers.VerifyResults(dstConn, targettable, 2, 1); - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + targettable); - Helpers.TryExecute(dstCmd, "drop table " + targetCustomerTable); } } + + Helpers.VerifyResults(dstConn, orderTable.Name, 2, 1); } } } - diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug85007.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug85007.cs index b3de348137..53501c3819 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug85007.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug85007.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,99 +17,90 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Bug85007", false); - string targetCustomerTable = dstTable + "_customer"; - using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try - { - Helpers.TryExecute(dstCmd, "CREATE TABLE [" + targetCustomerTable + "] ([CustomerID] [nchar] (5) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, CONSTRAINT [PK_" + targetCustomerTable + "] PRIMARY KEY CLUSTERED (CustomerID) ON [PRIMARY]) ON [PRIMARY]"); + // Declared before the order table so it is dropped last: the order table holds a + // foreign key referencing it. + using Table targetCustomerTable = new(dstConn, "SqlBulkCopyTest_Bug85007_customer", + "([CustomerID] [nchar] (5) COLLATE SQL_Latin1_General_CP1_CI_AS NOT NULL, PRIMARY KEY CLUSTERED (CustomerID) ON [PRIMARY]) ON [PRIMARY]"); - Helpers.TryExecute(dstCmd, - "CREATE TABLE [" + dstTable + "] (" + - " [OrderID] [int] IDENTITY (1, 1) NOT NULL ," + - " [CustomerID] [nchar] (5) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + - " [EmployeeID] [int] NULL ," + - " [OrderDate] [datetime] NULL ," + - " [RequiredDate] [datetime] NULL ," + - " [ShippedDate] [datetime] NULL ," + - " [ShipVia] [int] NULL ," + - " [Freight] [money] NULL CONSTRAINT [DF_" + dstTable + "_Freight] DEFAULT (0)," + - " [ShipName] [nvarchar] (40) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + - " [ShipAddress] [nvarchar] (60) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + - " [ShipCity] [nvarchar] (15) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + - " [ShipRegion] [nvarchar] (15) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + - " [ShipPostalCode] [nvarchar] (10) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + - " [ShipCountry] [nvarchar] (15) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + - " CONSTRAINT [PK_" + dstTable + "] PRIMARY KEY CLUSTERED " + - " (" + - " [OrderID]" + - " ) ON [PRIMARY] ," + - " CONSTRAINT [FK_" + dstTable + "_Customers] FOREIGN KEY " + - " (" + - " [CustomerID]" + - " ) REFERENCES [" + targetCustomerTable + "] (" + - " [CustomerID]" + - " )" + - ") ON [PRIMARY]"); + using Table dstTable = new(dstConn, "SqlBulkCopyTest_Bug85007", + "(" + + " [OrderID] [int] IDENTITY (1, 1) NOT NULL ," + + " [CustomerID] [nchar] (5) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + + " [EmployeeID] [int] NULL ," + + " [OrderDate] [datetime] NULL ," + + " [RequiredDate] [datetime] NULL ," + + " [ShippedDate] [datetime] NULL ," + + " [ShipVia] [int] NULL ," + + " [Freight] [money] NULL DEFAULT (0)," + + " [ShipName] [nvarchar] (40) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + + " [ShipAddress] [nvarchar] (60) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + + " [ShipCity] [nvarchar] (15) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + + " [ShipRegion] [nvarchar] (15) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + + " [ShipPostalCode] [nvarchar] (10) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + + " [ShipCountry] [nvarchar] (15) COLLATE SQL_Latin1_General_CP1_CI_AS NULL ," + + " PRIMARY KEY CLUSTERED " + + " (" + + " [OrderID]" + + " ) ON [PRIMARY] ," + + " FOREIGN KEY " + + " (" + + " [CustomerID]" + + " ) REFERENCES " + targetCustomerTable.Name + " (" + + " [CustomerID]" + + " )" + + ") ON [PRIMARY]"); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand customerCmd = new SqlCommand("SELECT CustomerID from Customers", srcConn)) - { - srcConn.Open(); + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand customerCmd = new SqlCommand("SELECT CustomerID from Customers", srcConn)) + { + srcConn.Open(); - // First copy the customer ID list across - using (DbDataReader reader = customerCmd.ExecuteReader()) + // First copy the customer ID list across + using (DbDataReader reader = customerCmd.ExecuteReader()) + { + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = targetCustomerTable; - bulkcopy.WriteToServer(reader); - } + bulkcopy.DestinationTableName = targetCustomerTable.Name; + bulkcopy.WriteToServer(reader); } + } - SqlCommand srcCmd = new SqlCommand("select * from orders", srcConn); - using (DbDataReader reader = srcCmd.ExecuteReader()) - { + using SqlCommand srcCmd = new SqlCommand("select * from orders", srcConn); + using (DbDataReader reader = srcCmd.ExecuteReader()) + { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - bulkcopy.BatchSize = 6; + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.BatchSize = 6; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("OrderID", "OrderID"); - ColumnMappings.Add("CustomerID", "CustomerID"); - ColumnMappings.Add("EmployeeID", "EmployeeID"); - ColumnMappings.Add("RequiredDate", "RequiredDate"); - ColumnMappings.Add("ShippedDate", "ShippedDate"); - ColumnMappings.Add("ShipVia", "ShipVia"); - ColumnMappings.Add("Freight", "Freight"); - ColumnMappings.Add("ShipName", "ShipName"); - ColumnMappings.Add("ShipAddress", "ShipAddress"); - ColumnMappings.Add("ShipCity", "ShipCity"); - ColumnMappings.Add("ShipRegion", "ShipRegion"); - ColumnMappings.Add("ShipPostalCode", "ShipPostalCode"); - ColumnMappings.Add("ShipCountry", "ShipCountry"); + ColumnMappings.Add("OrderID", "OrderID"); + ColumnMappings.Add("CustomerID", "CustomerID"); + ColumnMappings.Add("EmployeeID", "EmployeeID"); + ColumnMappings.Add("RequiredDate", "RequiredDate"); + ColumnMappings.Add("ShippedDate", "ShippedDate"); + ColumnMappings.Add("ShipVia", "ShipVia"); + ColumnMappings.Add("Freight", "Freight"); + ColumnMappings.Add("ShipName", "ShipName"); + ColumnMappings.Add("ShipAddress", "ShipAddress"); + ColumnMappings.Add("ShipCity", "ShipCity"); + ColumnMappings.Add("ShipRegion", "ShipRegion"); + ColumnMappings.Add("ShipPostalCode", "ShipPostalCode"); + ColumnMappings.Add("ShipCountry", "ShipCountry"); - bulkcopy.WriteToServer(reader); + bulkcopy.WriteToServer(reader); - DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 830, "Unexpected number of rows."); - } - Helpers.VerifyResults(dstConn, dstTable, 14, 830); + DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 830, "Unexpected number of rows."); } + Helpers.VerifyResults(dstConn, dstTable.Name, 14, 830); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - Helpers.TryExecute(dstCmd, "drop table " + targetCustomerTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug903514.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug903514.cs index d720fd9ebc..d07f9d8cac 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug903514.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug903514.cs @@ -1,10 +1,11 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,25 +17,13 @@ public class Bug903514 public void Test() { string constr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Bug903514", false); - using (SqlConnection dstConn = new SqlConnection(constr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) - { - dstConn.Open(); - - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 varchar(7000))"); - } + using SqlConnection dstConn = new SqlConnection(constr); + dstConn.Open(); - DoBulkCopy(constr, dstTable, 2); - DoBulkCopy(constr, dstTable, 0); + using Table dstTable = new(dstConn, "SqlBulkCopyTest_Bug903514", "(col1 int, col2 varchar(7000))"); - using (SqlConnection dstConn = new SqlConnection(constr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) - { - dstConn.Open(); - - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } + DoBulkCopy(constr, dstTable.Name, 2); + DoBulkCopy(constr, dstTable.Name, 0); } private static void DoBulkCopy(string dstConstr, string dstTable, int timeout) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug98182.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug98182.cs index f611e2e3b1..f9764b8de4 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug98182.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Bug98182.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -19,51 +20,38 @@ public void Test() string srctable = "[" + dstTable + " src]"; dstTable = "[" + dstTable + "]"; - string[] epilogue = { - "create table " + srctable + "([col 1] int primary key, [col 2] text)", - "insert into " + srctable + " values (33, 'Michael')", - "create table " + dstTable + "([col 1] int primary key, [col 2] text)", - }; - string[] prologue = { - "drop table " + srctable, - "drop table " + dstTable, - }; - using (SqlConnection dstConn = new SqlConnection(constr)) using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + + using Table srcTableObject = Table.WithName(dstConn, srctable, "([col 1] int primary key, [col 2] text)"); + using Table dstTableObject = Table.WithName(dstConn, dstTable, "([col 1] int primary key, [col 2] text)"); + + Helpers.TryExecute(dstCmd, "insert into " + srctable + " values (33, 'Michael')"); + + using (SqlConnection srcConn = new SqlConnection(constr)) + using (SqlCommand srcCmd = new SqlCommand(string.Format("select * from {0} ", srctable), srcConn)) { - Helpers.ProcessCommandBatch(typeof(SqlConnection), constr, epilogue); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(constr)) - using (SqlCommand srcCmd = new SqlCommand(string.Format("select * from {0} ", srctable), srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) { - srcConn.Open(); - - using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; + bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("[col 1]", "col 1"); - ColumnMappings.Add("col 2", "[col 2]"); + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + ColumnMappings.Add("[col 1]", "col 1"); + ColumnMappings.Add("col 2", "[col 2]"); - bulkcopy.WriteToServer(reader); + bulkcopy.WriteToServer(reader); - DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 1, "Unexpected number of rows."); - } - Helpers.VerifyResults(dstConn, dstTable, 2, 1); + DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 1, "Unexpected number of rows."); } + Helpers.VerifyResults(dstConn, dstTable, 2, 1); } } - finally - { - Helpers.ProcessCommandBatch(typeof(SqlConnection), constr, prologue); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CacheMetadata.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CacheMetadata.cs index 7443a99734..dc72a41a14 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CacheMetadata.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CacheMetadata.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Data; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -14,7 +15,7 @@ namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy public class CacheMetadata { private static readonly string sourceTable = "employees"; - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"; private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; // Test that CacheMetadata option works for multiple WriteToServer calls to the same table. @@ -23,55 +24,46 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadata", false); string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using SqlConnection dstConn = new(dstConstr); using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try + using Table dstTable = new(dstConn, "SqlBulkCopyTest_CacheMetadata", tableDefinition); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable.Name; + + // First WriteToServer: metadata is queried and cached. + using (SqlConnection srcConn = new(srcConstr)) + { + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); + } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 5); + + // Second WriteToServer: should reuse cached metadata. + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, initialQuery); - - using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); - bulkcopy.DestinationTableName = dstTable; - - // First WriteToServer: metadata is queried and cached. - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 5); - - // Second WriteToServer: should reuse cached metadata. - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 10); - - // Third WriteToServer: should still reuse cached metadata. - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 15); + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); } - finally + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 10); + + // Third WriteToServer: should still reuse cached metadata. + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 15); } } @@ -79,7 +71,7 @@ public void Test() public class CacheMetadataInvalidate { private static readonly string sourceTable = "employees"; - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"; private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; // Test that ClearCachedMetadata forces a fresh metadata query. @@ -88,47 +80,38 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataInvalidate", false); string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using SqlConnection dstConn = new(dstConstr); using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try + using Table dstTable = new(dstConn, "SqlBulkCopyTest_CacheMetadataInvalidate", tableDefinition); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable.Name; + + // First WriteToServer: metadata is queried and cached. + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, initialQuery); - - using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); - bulkcopy.DestinationTableName = dstTable; - - // First WriteToServer: metadata is queried and cached. - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 5); - - // Invalidate the cache and write again: should still succeed after re-querying metadata. - bulkcopy.ClearCachedMetadata(); - - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 10); + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); } - finally + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 5); + + // Invalidate the cache and write again: should still succeed after re-querying metadata. + bulkcopy.ClearCachedMetadata(); + + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 10); } } @@ -136,7 +119,7 @@ public void Test() public class CacheMetadataDestinationChange { private static readonly string sourceTable = "employees"; - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"; private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; // Test that changing DestinationTableName invalidates the cache and works correctly with a new table. @@ -145,50 +128,38 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable1 = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataDstChange0", false); - string dstTable2 = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataDstChange1", false); string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); - string initialQuery1 = string.Format(initialQueryTemplate, dstTable1); - string initialQuery2 = string.Format(initialQueryTemplate, dstTable2); using SqlConnection dstConn = new(dstConstr); using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try + using Table dstTable1 = new(dstConn, "SqlBulkCopyTest_CacheMetadataDstChange0", tableDefinition); + using Table dstTable2 = new(dstConn, "SqlBulkCopyTest_CacheMetadataDstChange1", tableDefinition); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + + // Write to first table. + bulkcopy.DestinationTableName = dstTable1.Name; + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, initialQuery1); - Helpers.TryExecute(dstCmd, initialQuery2); - - using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); - - // Write to first table. - bulkcopy.DestinationTableName = dstTable1; - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable1, 3, 5); - - // Change destination table: cache should be invalidated automatically. - bulkcopy.DestinationTableName = dstTable2; - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable2, 3, 5); + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); } - finally + Helpers.VerifyResults(dstConn, dstTable1.Name, 3, 5); + + // Change destination table: cache should be invalidated automatically. + bulkcopy.DestinationTableName = dstTable2.Name; + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryDropTable(dstConstr, dstTable1); - Helpers.TryDropTable(dstConstr, dstTable2); + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); } + Helpers.VerifyResults(dstConn, dstTable2.Name, 3, 5); } } @@ -196,7 +167,7 @@ public void Test() public class CacheMetadataWithoutFlag { private static readonly string sourceTable = "employees"; - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"; private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; // Test that without the CacheMetadata flag, multiple writes still work (no regression). @@ -205,60 +176,49 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataNoFlag", false); string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using SqlConnection dstConn = new(dstConstr); using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try + using Table dstTable = new(dstConn, "SqlBulkCopyTest_CacheMetadataNoFlag", tableDefinition); + + using SqlBulkCopy bulkcopy = new(dstConn); + bulkcopy.DestinationTableName = dstTable.Name; + + // First WriteToServer without CacheMetadata. + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, initialQuery); - - using SqlBulkCopy bulkcopy = new(dstConn); - bulkcopy.DestinationTableName = dstTable; - - // First WriteToServer without CacheMetadata. - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 5); - - // Second WriteToServer without CacheMetadata. - using (SqlConnection srcConn = new(srcConstr)) - { - srcConn.Open(); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = srcCmd.ExecuteReader(); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 10); + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); } - finally + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 5); + + // Second WriteToServer without CacheMetadata. + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + srcConn.Open(); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = srcCmd.ExecuteReader(); + bulkcopy.WriteToServer(reader); } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 10); } } [Trait("Set", "2")] public class CacheMetadataWithDataTable { - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50), col3 nvarchar(50))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(50), col3 nvarchar(50))"; // Test that CacheMetadata works with DataTable source as well as IDataReader. [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] public void Test() { string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataDT", false); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using DataTable sourceData = new(); sourceData.Columns.Add("col1", typeof(int)); @@ -272,32 +232,25 @@ public void Test() using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try - { - Helpers.TryExecute(dstCmd, initialQuery); + using Table dstTable = new(dstConn, "SqlBulkCopyTest_CacheMetadataDT", tableDefinition); - using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); - bulkcopy.DestinationTableName = dstTable; + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable.Name; - // First WriteToServer with DataTable: metadata is queried and cached. - bulkcopy.WriteToServer(sourceData); - Helpers.VerifyResults(dstConn, dstTable, 3, 3); + // First WriteToServer with DataTable: metadata is queried and cached. + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 3); - // Second WriteToServer with DataTable: should reuse cached metadata. - bulkcopy.WriteToServer(sourceData); - Helpers.VerifyResults(dstConn, dstTable, 3, 6); - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } + // Second WriteToServer with DataTable: should reuse cached metadata. + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 6); } } [Trait("Set", "2")] public class CacheMetadataColumnMappingsChange { - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50), col3 nvarchar(50))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(50), col3 nvarchar(50))"; // Test that changing ColumnMappings between WriteToServer calls works correctly with CacheMetadata. // The cached metadata describes the destination table schema, not the column mappings, @@ -306,8 +259,6 @@ public class CacheMetadataColumnMappingsChange public void Test() { string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataColMap", false); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using DataTable sourceData = new DataTable(); sourceData.Columns.Add("id", typeof(int)); @@ -320,52 +271,45 @@ public void Test() using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try + using Table dstTable = new(dstConn, "SqlBulkCopyTest_CacheMetadataColMap", tableDefinition); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable.Name; + + // First write: map firstName -> col2, lastName -> col3. + bulkcopy.ColumnMappings.Add("id", "col1"); + bulkcopy.ColumnMappings.Add("firstName", "col2"); + bulkcopy.ColumnMappings.Add("lastName", "col3"); + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 2); + + // Verify first mapping: col2 should contain firstName values. + using (SqlCommand verifyCmd = new("select col2 from " + dstTable.Name + " where col1 = 1", dstConn)) { - Helpers.TryExecute(dstCmd, initialQuery); - - using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); - bulkcopy.DestinationTableName = dstTable; - - // First write: map firstName -> col2, lastName -> col3. - bulkcopy.ColumnMappings.Add("id", "col1"); - bulkcopy.ColumnMappings.Add("firstName", "col2"); - bulkcopy.ColumnMappings.Add("lastName", "col3"); - bulkcopy.WriteToServer(sourceData); - Helpers.VerifyResults(dstConn, dstTable, 3, 2); - - // Verify first mapping: col2 should contain firstName values. - using (SqlCommand verifyCmd = new("select col2 from " + dstTable + " where col1 = 1", dstConn)) - { - object result = verifyCmd.ExecuteScalar(); - Assert.Equal("Alice", result); - } - - // Change mappings: swap col2 and col3 targets. - bulkcopy.ColumnMappings.Clear(); - bulkcopy.ColumnMappings.Add("id", "col1"); - bulkcopy.ColumnMappings.Add("firstName", "col3"); - bulkcopy.ColumnMappings.Add("lastName", "col2"); - bulkcopy.WriteToServer(sourceData); - Helpers.VerifyResults(dstConn, dstTable, 3, 4); - - // Verify second mapping: col3 should now contain firstName values for the new rows. - using (SqlCommand verifyCmd = new("select col3 from " + dstTable + " where col1 = 1 order by col2", dstConn)) - { - using SqlDataReader reader = verifyCmd.ExecuteReader(); - - // First row (from first write): col3 = "Smith" (lastName). - Assert.True(reader.Read()); - Assert.Equal("Smith", reader.GetString(0)); - - // Second row (from second write): col3 = "Alice" (firstName). - Assert.True(reader.Read()); - Assert.Equal("Alice", reader.GetString(0)); - } + object result = verifyCmd.ExecuteScalar(); + Assert.Equal("Alice", result); } - finally + + // Change mappings: swap col2 and col3 targets. + bulkcopy.ColumnMappings.Clear(); + bulkcopy.ColumnMappings.Add("id", "col1"); + bulkcopy.ColumnMappings.Add("firstName", "col3"); + bulkcopy.ColumnMappings.Add("lastName", "col2"); + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 4); + + // Verify second mapping: col3 should now contain firstName values for the new rows. + using (SqlCommand verifyCmd = new("select col3 from " + dstTable.Name + " where col1 = 1 order by col2", dstConn)) { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + using SqlDataReader reader = verifyCmd.ExecuteReader(); + + // First row (from first write): col3 = "Smith" (lastName). + Assert.True(reader.Read()); + Assert.Equal("Smith", reader.GetString(0)); + + // Second row (from second write): col3 = "Alice" (firstName). + Assert.True(reader.Read()); + Assert.Equal("Alice", reader.GetString(0)); } } } @@ -373,7 +317,7 @@ public void Test() [Trait("Set", "2")] public class CacheMetadataColumnSubsetChange { - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50), col3 nvarchar(50))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(50), col3 nvarchar(50))"; // Test that mapping a subset of columns on the first call, then all columns on the // second call, works correctly with CacheMetadata. This verifies that null-pruning of @@ -383,8 +327,6 @@ public class CacheMetadataColumnSubsetChange public void Test() { string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataSubset", false); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using DataTable sourceData = new DataTable(); sourceData.Columns.Add("id", typeof(int)); @@ -397,39 +339,32 @@ public void Test() using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try - { - Helpers.TryExecute(dstCmd, initialQuery); - - using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); - bulkcopy.DestinationTableName = dstTable; - - // First write: map only col1 and col2 (col3 is unmatched and will be pruned). - bulkcopy.ColumnMappings.Add("id", "col1"); - bulkcopy.ColumnMappings.Add("firstName", "col2"); - bulkcopy.WriteToServer(sourceData); - Helpers.VerifyResults(dstConn, dstTable, 3, 2); - - // Second write: map all three columns including col3. - // Without the clone fix, this would fail because col3 metadata was - // permanently nulled in the cache during the first call. - bulkcopy.ColumnMappings.Clear(); - bulkcopy.ColumnMappings.Add("id", "col1"); - bulkcopy.ColumnMappings.Add("firstName", "col2"); - bulkcopy.ColumnMappings.Add("lastName", "col3"); - bulkcopy.WriteToServer(sourceData); - Helpers.VerifyResults(dstConn, dstTable, 3, 4); - - // Verify col3 has the expected data from the second write. - using (SqlCommand verifyCmd = new("select col3 from " + dstTable + " where col1 = 1 and col3 is not null", dstConn)) - { - object result = verifyCmd.ExecuteScalar(); - Assert.Equal("Smith", result); - } - } - finally + using Table dstTable = new(dstConn, "SqlBulkCopyTest_CacheMetadataSubset", tableDefinition); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable.Name; + + // First write: map only col1 and col2 (col3 is unmatched and will be pruned). + bulkcopy.ColumnMappings.Add("id", "col1"); + bulkcopy.ColumnMappings.Add("firstName", "col2"); + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 2); + + // Second write: map all three columns including col3. + // Without the clone fix, this would fail because col3 metadata was + // permanently nulled in the cache during the first call. + bulkcopy.ColumnMappings.Clear(); + bulkcopy.ColumnMappings.Add("id", "col1"); + bulkcopy.ColumnMappings.Add("firstName", "col2"); + bulkcopy.ColumnMappings.Add("lastName", "col3"); + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 4); + + // Verify col3 has the expected data from the second write. + using (SqlCommand verifyCmd = new("select col3 from " + dstTable.Name + " where col1 = 1 and col3 is not null", dstConn)) { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + object result = verifyCmd.ExecuteScalar(); + Assert.Equal("Smith", result); } } } @@ -438,7 +373,7 @@ public void Test() public class CacheMetadataAsync { private static readonly string sourceTable = "employees"; - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"; private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; // Test that CacheMetadata works correctly with WriteToServerAsync. @@ -447,77 +382,66 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataAsync", false); - Task t = TestAsync(srcConstr, dstConstr, dstTable); + Task t = TestAsync(srcConstr, dstConstr); t.Wait(); Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); } - private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable) + private static async Task TestAsync(string srcConstr, string dstConstr) { string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using SqlConnection dstConn = new(dstConstr); using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try + using Table dstTable = new(dstConn, "SqlBulkCopyTest_CacheMetadataAsync", tableDefinition); + + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); + bulkcopy.DestinationTableName = dstTable.Name; + + // First WriteToServerAsync: metadata is queried and cached. + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, initialQuery); - - using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata, null); - bulkcopy.DestinationTableName = dstTable; - - // First WriteToServerAsync: metadata is queried and cached. - using (SqlConnection srcConn = new(srcConstr)) - { - await srcConn.OpenAsync().ConfigureAwait(false); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); - await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 5); - - // Second WriteToServerAsync: should reuse cached metadata. - using (SqlConnection srcConn = new(srcConstr)) - { - await srcConn.OpenAsync().ConfigureAwait(false); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); - await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 10); - - // Third WriteToServerAsync: should still reuse cached metadata. - using (SqlConnection srcConn = new(srcConstr)) - { - await srcConn.OpenAsync().ConfigureAwait(false); - using SqlCommand srcCmd = new(sourceQuery, srcConn); - using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); - await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 15); + await srcConn.OpenAsync().ConfigureAwait(false); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); + await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); } - finally + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 5); + + // Second WriteToServerAsync: should reuse cached metadata. + using (SqlConnection srcConn = new(srcConstr)) + { + await srcConn.OpenAsync().ConfigureAwait(false); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); + await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); + } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 10); + + // Third WriteToServerAsync: should still reuse cached metadata. + using (SqlConnection srcConn = new(srcConstr)) { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + await srcConn.OpenAsync().ConfigureAwait(false); + using SqlCommand srcCmd = new(sourceQuery, srcConn); + using IDataReader reader = await srcCmd.ExecuteReaderAsync().ConfigureAwait(false); + await bulkcopy.WriteToServerAsync(reader).ConfigureAwait(false); } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 15); } } [Trait("Set", "2")] public class CacheMetadataCombinedWithKeepNulls { - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(50) default 'DefaultVal', col3 nvarchar(50))"; + private static readonly string tableDefinition = "(col1 int, col2 nvarchar(50) default 'DefaultVal', col3 nvarchar(50))"; // Test that CacheMetadata works correctly when combined with other SqlBulkCopyOptions. [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] public void Test() { string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CacheMetadataKeepNulls", false); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using DataTable sourceData = new(); sourceData.Columns.Add("col1", typeof(int)); @@ -530,33 +454,26 @@ public void Test() using SqlCommand dstCmd = dstConn.CreateCommand(); dstConn.Open(); - try - { - Helpers.TryExecute(dstCmd, initialQuery); + using Table dstTable = new(dstConn, "SqlBulkCopyTest_CacheMetadataKeepNulls", tableDefinition); - using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata | SqlBulkCopyOptions.KeepNulls, null); - bulkcopy.DestinationTableName = dstTable; - bulkcopy.ColumnMappings.Add("col1", "col1"); - bulkcopy.ColumnMappings.Add("col2", "col2"); - bulkcopy.ColumnMappings.Add("col3", "col3"); + using SqlBulkCopy bulkcopy = new(dstConn, SqlBulkCopyOptions.CacheMetadata | SqlBulkCopyOptions.KeepNulls, null); + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.ColumnMappings.Add("col1", "col1"); + bulkcopy.ColumnMappings.Add("col2", "col2"); + bulkcopy.ColumnMappings.Add("col3", "col3"); - // First write with CacheMetadata | KeepNulls. - bulkcopy.WriteToServer(sourceData); - Helpers.VerifyResults(dstConn, dstTable, 3, 2); + // First write with CacheMetadata | KeepNulls. + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 2); - // Verify nulls were kept (not replaced by default values). - using SqlCommand verifyCmd = new("select col2 from " + dstTable + " where col1 = 1", dstConn); - object result = verifyCmd.ExecuteScalar(); - Assert.Equal(System.DBNull.Value, result); + // Verify nulls were kept (not replaced by default values). + using SqlCommand verifyCmd = new("select col2 from " + dstTable.Name + " where col1 = 1", dstConn); + object result = verifyCmd.ExecuteScalar(); + Assert.Equal(System.DBNull.Value, result); - // Second write should reuse cached metadata. - bulkcopy.WriteToServer(sourceData); - Helpers.VerifyResults(dstConn, dstTable, 3, 4); - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } + // Second write should reuse cached metadata. + bulkcopy.WriteToServer(sourceData); + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 4); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CheckConstraints.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CheckConstraints.cs index ede548f9cf..f99b37c98a 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CheckConstraints.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CheckConstraints.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -15,54 +16,44 @@ public class CheckConstraints public void Test() { string constr = DataTestUtility.TCPConnectionString; - string srctable = DataTestUtility.GetShortName("SqlBulkCopyTest_Extensionsrc", false); - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Extensiondst", false); using (SqlConnection dstConn = new SqlConnection(constr)) using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try - { - // create the source table - Helpers.TryExecute(dstCmd, "create table " + srctable + " (col1 int , col2 int, col3 text)"); - Helpers.TryExecute(dstCmd, "insert into " + srctable + " values (33, 498, 'Michael')"); - Helpers.TryExecute(dstCmd, "insert into " + srctable + " values (34, 499, 'Astrid')"); - Helpers.TryExecute(dstCmd, "insert into " + srctable + " values (65, 500, 'alles Käse')"); - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int primary key, col2 int CONSTRAINT CK_" + dstTable + " CHECK (col2 < 500), col3 text)"); + using Table srctable = new(dstConn, "SqlBulkCopyTest_Extensionsrc", "(col1 int , col2 int, col3 text)"); + using Table dstTable = new(dstConn, "SqlBulkCopyTest_Extensiondst", "(col1 int primary key, col2 int CHECK (col2 < 500), col3 text)"); + + Helpers.TryExecute(dstCmd, "insert into " + srctable.Name + " values (33, 498, 'Michael')"); + Helpers.TryExecute(dstCmd, "insert into " + srctable.Name + " values (34, 499, 'Astrid')"); + Helpers.TryExecute(dstCmd, "insert into " + srctable.Name + " values (65, 500, 'alles Käse')"); - using (SqlConnection srcConn = new SqlConnection(constr)) - using (SqlCommand srcCmd = new SqlCommand("select * from " + srctable, srcConn)) + using (SqlConnection srcConn = new SqlConnection(constr)) + using (SqlCommand srcCmd = new SqlCommand("select * from " + srctable.Name, srcConn)) + { + srcConn.Open(); + using (DbDataReader reader = srcCmd.ExecuteReader()) { - srcConn.Open(); - using (DbDataReader reader = srcCmd.ExecuteReader()) + try { - try + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.CheckConstraints, null)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.CheckConstraints, null)) - { - bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + bulkcopy.DestinationTableName = dstTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("col1", "col1"); - ColumnMappings.Add("col2", "col2"); - ColumnMappings.Add("col3", "col3"); - bulkcopy.WriteToServer(reader); - } - } - catch (SqlException sqlEx) - { - // Error 547 == The %ls statement conflicted with the %ls constraint "%.*ls". - DataTestUtility.AssertEqualsWithDescription(547, sqlEx.Number, "Unexpected error number."); + ColumnMappings.Add("col1", "col1"); + ColumnMappings.Add("col2", "col2"); + ColumnMappings.Add("col3", "col3"); + bulkcopy.WriteToServer(reader); } } + catch (SqlException sqlEx) + { + // Error 547 == The %ls statement conflicted with the %ls constraint "%.*ls". + DataTestUtility.AssertEqualsWithDescription(547, sqlEx.Number, "Unexpected error number."); + } } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - Helpers.TryExecute(dstCmd, "drop table " + srctable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/ColumnCollation.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/ColumnCollation.cs index 4cffae2fb2..9ffebcd0a2 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/ColumnCollation.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/ColumnCollation.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -15,14 +16,12 @@ public class ColumnCollation public void Test() { string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_ColumnCollation", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (name_jp varchar(20) collate Japanese_CI_AS, " + - "name_ru varchar(20) collate Cyrillic_General_CI_AS)"); + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_ColumnCollation", + "(name_jp varchar(20) collate Japanese_CI_AS, name_ru varchar(20) collate Cyrillic_General_CI_AS)"); string s_jp = "江戸糸あやつり人形"; string s_ru = "проверка"; @@ -37,11 +36,11 @@ public void Test() using (SqlBulkCopy bcp = new SqlBulkCopy(dstConn)) { - bcp.DestinationTableName = dstTable; + bcp.DestinationTableName = dstTable.Name; bcp.WriteToServer(table); } - using (SqlDataReader reader = (new SqlCommand("select * from " + dstTable, dstConn)).ExecuteReader()) + using (SqlDataReader reader = (new SqlCommand("select * from " + dstTable.Name, dstConn)).ExecuteReader()) { while (reader.Read()) { @@ -54,16 +53,7 @@ public void Test() "Unexpected value: " + reader["name_ru"]); } } - - } - - using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) - { - dstConn.Open(); - Helpers.TryExecute(dstCmd, "drop table " + dstTable); } } - } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs index 7c82e6dd21..6113fadcb1 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Data.Common; using System.Diagnostics; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -13,73 +14,61 @@ namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy [Trait("Set", "2")] public class CopyAllFromReader { - private static readonly string destinationTable = null; private static readonly string sourceTable = "employees"; - private static readonly string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + private const string TableDefinition = "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"; private static readonly string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CopyAllFromReader", false); Debug.Assert((int)SqlBulkCopyOptions.UseInternalTransaction == 1 << 5, "Compiler screwed up the options"); - dstTable = destinationTable != null ? destinationTable : dstTable; - string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_CopyAllFromReader", TableDefinition); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) { - Helpers.TryExecute(dstCmd, initialQuery); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) - { - srcConn.Open(); + srcConn.Open(); - using (DbDataReader reader = srcCmd.ExecuteReader()) + using (DbDataReader reader = srcCmd.ExecuteReader()) + { + IDictionary stats; + long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 4 : 13; + long expectedSelectRows = DataTestUtility.IsAzureSynapse ? 4 : 15; + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - IDictionary stats; - long expectedSelectCount = DataTestUtility.IsAzureSynapse ? 4 : 13; - long expectedSelectRows = DataTestUtility.IsAzureSynapse ? 4 : 15; - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - dstConn.StatisticsEnabled = true; - bulkcopy.WriteToServer(reader); - dstConn.StatisticsEnabled = false; - stats = dstConn.RetrieveStatistics(); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 5); + bulkcopy.DestinationTableName = dstTable.Name; + dstConn.StatisticsEnabled = true; + bulkcopy.WriteToServer(reader); + dstConn.StatisticsEnabled = false; + stats = dstConn.RetrieveStatistics(); + } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 5); - Assert.True(0 < (long)stats["BytesReceived"], "BytesReceived is non-positive."); - Assert.True(0 < (long)stats["BytesSent"], "BytesSent is non-positive."); - DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["UnpreparedExecs"], "Non-zero UnpreparedExecs value: " + (long)stats["UnpreparedExecs"]); - DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["PreparedExecs"], "Non-zero PreparedExecs value: " + (long)stats["PreparedExecs"]); - DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["Prepares"], "Non-zero Prepares value: " + (long)stats["Prepares"]); - DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["CursorOpens"], "Non-zero CursorOpens value: " + (long)stats["CursorOpens"]); - DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["IduRows"], "Non-zero IduRows value: " + (long)stats["IduRows"]); + Assert.True(0 < (long)stats["BytesReceived"], "BytesReceived is non-positive."); + Assert.True(0 < (long)stats["BytesSent"], "BytesSent is non-positive."); + DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["UnpreparedExecs"], "Non-zero UnpreparedExecs value: " + (long)stats["UnpreparedExecs"]); + DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["PreparedExecs"], "Non-zero PreparedExecs value: " + (long)stats["PreparedExecs"]); + DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["Prepares"], "Non-zero Prepares value: " + (long)stats["Prepares"]); + DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["CursorOpens"], "Non-zero CursorOpens value: " + (long)stats["CursorOpens"]); + DataTestUtility.AssertEqualsWithDescription((long)0, (long)stats["IduRows"], "Non-zero IduRows value: " + (long)stats["IduRows"]); - DataTestUtility.AssertEqualsWithDescription((long)3, stats["BuffersReceived"], "Unexpected BuffersReceived value."); - DataTestUtility.AssertEqualsWithDescription((long)3, stats["BuffersSent"], "Unexpected BuffersSent value."); - DataTestUtility.AssertEqualsWithDescription((long)0, stats["IduCount"], "Unexpected IduCount value."); - DataTestUtility.AssertEqualsWithDescription(expectedSelectCount, stats["SelectCount"], "Unexpected SelectCount value."); - DataTestUtility.AssertEqualsWithDescription((long)3, stats["ServerRoundtrips"], "Unexpected ServerRoundtrips value."); - DataTestUtility.AssertEqualsWithDescription(expectedSelectRows, stats["SelectRows"], "Unexpected SelectRows value."); - DataTestUtility.AssertEqualsWithDescription((long)2, stats["SumResultSets"], "Unexpected SumResultSets value."); - DataTestUtility.AssertEqualsWithDescription((long)0, stats["Transactions"], "Unexpected Transactions value."); - } + DataTestUtility.AssertEqualsWithDescription((long)3, stats["BuffersReceived"], "Unexpected BuffersReceived value."); + DataTestUtility.AssertEqualsWithDescription((long)3, stats["BuffersSent"], "Unexpected BuffersSent value."); + DataTestUtility.AssertEqualsWithDescription((long)0, stats["IduCount"], "Unexpected IduCount value."); + DataTestUtility.AssertEqualsWithDescription(expectedSelectCount, stats["SelectCount"], "Unexpected SelectCount value."); + DataTestUtility.AssertEqualsWithDescription((long)3, stats["ServerRoundtrips"], "Unexpected ServerRoundtrips value."); + DataTestUtility.AssertEqualsWithDescription(expectedSelectRows, stats["SelectRows"], "Unexpected SelectRows value."); + DataTestUtility.AssertEqualsWithDescription((long)2, stats["SumResultSets"], "Unexpected SumResultSets value."); + DataTestUtility.AssertEqualsWithDescription((long)0, stats["Transactions"], "Unexpected Transactions value."); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader1.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader1.cs index 78bc18f17a..0b731df803 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader1.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReader1.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,43 +17,34 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CopyAllFromReader1", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try - { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_CopyAllFromReader1", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 * from employees", srcConn)) + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 * from employees", srcConn)) + { + srcConn.Open(); + using (DbDataReader reader = srcCmd.ExecuteReader()) { - srcConn.Open(); - using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + bulkcopy.DestinationTableName = dstTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("EmployeeID", "col1"); - ColumnMappings.Add("LastName", "col2"); - ColumnMappings.Add("FirstName", "col3"); + ColumnMappings.Add("EmployeeID", "col1"); + ColumnMappings.Add("LastName", "col2"); + ColumnMappings.Add("FirstName", "col3"); - bulkcopy.WriteToServer(reader); + bulkcopy.WriteToServer(reader); - DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 5, "Unexpected number of rows."); - DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied64, (long)5, "Unexpected number of rows."); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 5); + DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 5, "Unexpected number of rows."); + DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied64, (long)5, "Unexpected number of rows."); } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 5); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderAsync.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderAsync.cs index d9cc530472..ba1c4f0bbf 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderAsync.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderAsync.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -18,56 +19,46 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_AsyncTest1", false); // Use this semaphore to ensure that results are written to the log in the correct order SemaphoreSlim outputSemaphore = new SemaphoreSlim(0, 1); - Task t = TestAsync(srcConstr, dstConstr, dstTable, outputSemaphore); + Task t = TestAsync(srcConstr, dstConstr, outputSemaphore); outputSemaphore.Release(); t.Wait(); Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); } - private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable, SemaphoreSlim outputSemaphore) + private static async Task TestAsync(string srcConstr, string dstConstr, SemaphoreSlim outputSemaphore) { - string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; string srcTable = "employees"; string sourceQuery = string.Format(sourceQueryTemplate, srcTable); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_AsyncTest1", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) { - Helpers.TryExecute(dstCmd, initialQuery); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) + srcConn.Open(); + using (DbDataReader reader = srcCmd.ExecuteReader()) { - srcConn.Open(); - using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; + bulkcopy.DestinationTableName = dstTable.Name; - await bulkcopy.WriteToServerAsync(reader); - await outputSemaphore.WaitAsync(); + await bulkcopy.WriteToServerAsync(reader); + await outputSemaphore.WaitAsync(); - DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 5, "Unexpected number of rows."); - DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied64, (long)5, "Unexpected number of rows."); - } - Helpers.VerifyResults(dstConn, dstTable, 3, 5); + DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied, 5, "Unexpected number of rows."); + DataTestUtility.AssertEqualsWithDescription(bulkcopy.RowsCopied64, (long)5, "Unexpected number of rows."); } + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 5); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderCancelAsync.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderCancelAsync.cs index 4329f7f5f4..0ed67a7aba 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderCancelAsync.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderCancelAsync.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -20,47 +21,37 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_AsyncTest5", false); cts = new CancellationTokenSource(); cts.Cancel(); - Task t = TestAsync(srcConstr, dstConstr, dstTable, cts.Token); + Task t = TestAsync(srcConstr, dstConstr, cts.Token); DataTestUtility.AssertThrowsInner(() => t.Wait()); Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); } - private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable, CancellationToken ctoken) + private static async Task TestAsync(string srcConstr, string dstConstr, CancellationToken ctoken) { - string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; string srcTable = "employees"; string sourceQuery = string.Format(sourceQueryTemplate, srcTable); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_AsyncTest5", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) { - Helpers.TryExecute(dstCmd, initialQuery); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) - { - srcConn.Open(); + srcConn.Open(); - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - await bulkcopy.WriteToServerAsync(reader, ctoken); - } + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = dstTable.Name; + await bulkcopy.WriteToServerAsync(reader, ctoken); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderConnectionCloseAsync.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderConnectionCloseAsync.cs index 44b9c89dd7..daf14f4c95 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderConnectionCloseAsync.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderConnectionCloseAsync.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Data.Common; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -18,46 +19,38 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_AsyncTest6", false); - Task t = TestAsync(srcConstr, dstConstr, dstTable); + Task t = TestAsync(srcConstr, dstConstr); DataTestUtility.AssertThrowsInner(() => t.Wait()); Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); } - private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable) + private static async Task TestAsync(string srcConstr, string dstConstr) { - string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10))"; + string tableDefinition = "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"; string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; string sourceTable = "employees"; string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); - string initialQuery = string.Format(initialQueryTemplate, dstTable); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + + using Table dstTable = new(dstConn, "SqlBulkCopyTest_AsyncTest6", tableDefinition); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) { - Helpers.TryExecute(dstCmd, initialQuery); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) - { - srcConn.Open(); + srcConn.Open(); - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - dstConn.Close(); - await bulkcopy.WriteToServerAsync(reader); - } + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = dstTable.Name; + dstConn.Close(); + await bulkcopy.WriteToServerAsync(reader); } } - finally - { - Helpers.TryDropTable(dstConstr, dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderConnectionCloseOnEventAsync.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderConnectionCloseOnEventAsync.cs index e48bf5906a..78e75a4f37 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderConnectionCloseOnEventAsync.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyAllFromReaderConnectionCloseOnEventAsync.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Data.Common; using System.IO; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -13,61 +14,53 @@ namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy [Trait("Set", "2")] public class CopyAllFromReaderConnectionClosedOnEventAsync { - [Trait("Category", "flaky")] // Hangs and crashes on occasion + [Trait("category", "flaky")] // Hangs and crashes on occasion [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_AsyncTest7", false); #if DEBUG - string initialQueryTemplate = "create table {0} (col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 varchar(8000))"; + string tableDefinition = "(col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 varchar(8000))"; string sourceQuery = "select EmployeeID, LastName, FirstName, REPLICATE('a', 8000) from employees"; - string initialQuery = string.Format(initialQueryTemplate, dstTable); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + + using Table dstTable = new(dstConn, "SqlBulkCopyTest_AsyncTest7", tableDefinition); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) { - Helpers.TryExecute(dstCmd, initialQuery); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) - { - srcConn.Open(); + srcConn.Open(); - using (DbDataReader reader = srcCmd.ExecuteReader()) + using (DbDataReader reader = srcCmd.ExecuteReader()) + { + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; + bulkcopy.DestinationTableName = dstTable.Name; - // Close the bulk copy's connection when it notifies us - bulkcopy.NotifyAfter = 1; - bulkcopy.SqlRowsCopied += (sender, e) => - { - dstConn.Close(); - }; + // Close the bulk copy's connection when it notifies us + bulkcopy.NotifyAfter = 1; + bulkcopy.SqlRowsCopied += (sender, e) => + { + dstConn.Close(); + }; - using (AsyncDebugScope debugScope = new AsyncDebugScope()) - { - // Force all writes to pend, this will guarantee that we will go through the correct code path - debugScope.ForceAsyncWriteDelay = 1; + using (AsyncDebugScope debugScope = new AsyncDebugScope()) + { + // Force all writes to pend, this will guarantee that we will go through the correct code path + debugScope.ForceAsyncWriteDelay = 1; - // Check that the copying fails - string message = string.Format(SystemDataResourceManager.Instance.ADP_OpenConnectionRequired, "WriteToServer", SystemDataResourceManager.Instance.ADP_ConnectionStateMsg_Closed); - DataTestUtility.AssertThrowsInnerWithAlternate(() => bulkcopy.WriteToServerAsync(reader).Wait(5000), innerExceptionMessage: message); - } + // Check that the copying fails + string message = string.Format(SystemDataResourceManager.Instance.ADP_OpenConnectionRequired, "WriteToServer", SystemDataResourceManager.Instance.ADP_ConnectionStateMsg_Closed); + DataTestUtility.AssertThrowsInnerWithAlternate(() => bulkcopy.WriteToServerAsync(reader).Wait(5000), innerExceptionMessage: message); } } } } - finally - { - Helpers.TryDropTable(dstConstr, dstTable); - } } #endif } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyMultipleReaders.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyMultipleReaders.cs index 043c0f75f5..7a670d207b 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyMultipleReaders.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyMultipleReaders.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,53 +17,44 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CopyMultipleReaders", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_CopyMultipleReaders", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = srcConn.CreateCommand()) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = srcConn.CreateCommand()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - srcConn.Open(); - - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + bulkcopy.DestinationTableName = dstTable.Name; + srcCmd.CommandText = "select EmployeeID, LastName from employees where LastName < 'E%'"; + using (DbDataReader reader = srcCmd.ExecuteReader()) { - bulkcopy.DestinationTableName = dstTable; - srcCmd.CommandText = "select EmployeeID, LastName from employees where LastName < 'E%'"; - using (DbDataReader reader = srcCmd.ExecuteReader()) - { - bulkcopy.WriteToServer(reader); - } - DataTestUtility.AssertEqualsWithDescription(0, bulkcopy.ColumnMappings.Count, "Unexpected ColumnMappings count."); - - srcCmd.CommandText = "select EmployeeID, LastName, FirstName from employees where LastName > 'D%'"; - using (DbDataReader reader = srcCmd.ExecuteReader()) - { - bulkcopy.WriteToServer(reader); - } - DataTestUtility.AssertEqualsWithDescription(0, bulkcopy.ColumnMappings.Count, "Unexpected ColumnMappings count."); + bulkcopy.WriteToServer(reader); + } + DataTestUtility.AssertEqualsWithDescription(0, bulkcopy.ColumnMappings.Count, "Unexpected ColumnMappings count."); - srcCmd.CommandText = "select EmployeeID, FirstName from employees where LastName < 'E%'"; - using (DbDataReader reader = srcCmd.ExecuteReader()) - { - bulkcopy.WriteToServer(reader); - } - DataTestUtility.AssertEqualsWithDescription(0, bulkcopy.ColumnMappings.Count, "Unexpected ColumnMappings count."); + srcCmd.CommandText = "select EmployeeID, LastName, FirstName from employees where LastName > 'D%'"; + using (DbDataReader reader = srcCmd.ExecuteReader()) + { + bulkcopy.WriteToServer(reader); + } + DataTestUtility.AssertEqualsWithDescription(0, bulkcopy.ColumnMappings.Count, "Unexpected ColumnMappings count."); - Helpers.VerifyResults(dstConn, dstTable, 3, 15); + srcCmd.CommandText = "select EmployeeID, FirstName from employees where LastName < 'E%'"; + using (DbDataReader reader = srcCmd.ExecuteReader()) + { + bulkcopy.WriteToServer(reader); } + DataTestUtility.AssertEqualsWithDescription(0, bulkcopy.ColumnMappings.Count, "Unexpected ColumnMappings count."); + + Helpers.VerifyResults(dstConn, dstTable.Name, 3, 15); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromDatatable.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromDatatable.cs index 324ddad5c8..55b7f34152 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromDatatable.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromDatatable.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,64 +17,56 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CopySomeFromDataTable", false); DataSet dataset; SqlDataAdapter adapter; DataTable datatable; using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try - { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) - { - srcConn.Open(); + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_CopySomeFromDataTable", "(col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); - dataset = new DataSet("MyDataSet"); - adapter = new SqlDataAdapter(srcCmd); - adapter.Fill(dataset); - datatable = dataset.Tables[0]; + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) + { + srcConn.Open(); - string columnname; + dataset = new DataSet("MyDataSet"); + adapter = new SqlDataAdapter(srcCmd); + adapter.Fill(dataset); + datatable = dataset.Tables[0]; - foreach (DataColumn column in datatable.Columns) - { - columnname = column.ColumnName; - } + string columnname; - datatable.Rows[0].BeginEdit(); - datatable.Rows[0][0] = 333; - datatable.Rows[0].EndEdit(); + foreach (DataColumn column in datatable.Columns) + { + columnname = column.ColumnName; + } - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - bulkcopy.BatchSize = 7; + datatable.Rows[0].BeginEdit(); + datatable.Rows[0][0] = 333; + datatable.Rows[0].EndEdit(); - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.BatchSize = 7; - ColumnMappings.Add(0, "col1"); - ColumnMappings.Add(1, "col2"); - ColumnMappings.Add(2, "col3"); - bulkcopy.WriteToServer(datatable, DataRowState.Unchanged); - datatable.Rows.GetEnumerator().Reset(); - bulkcopy.WriteToServer(datatable, DataRowState.Modified); - datatable.Rows.GetEnumerator().Reset(); - bulkcopy.WriteToServer(datatable, DataRowState.Deleted); - bulkcopy.Close(); - } + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - Helpers.VerifyResults(dstConn, dstTable, 4, 18); + ColumnMappings.Add(0, "col1"); + ColumnMappings.Add(1, "col2"); + ColumnMappings.Add(2, "col3"); + bulkcopy.WriteToServer(datatable, DataRowState.Unchanged); + datatable.Rows.GetEnumerator().Reset(); + bulkcopy.WriteToServer(datatable, DataRowState.Modified); + datatable.Rows.GetEnumerator().Reset(); + bulkcopy.WriteToServer(datatable, DataRowState.Deleted); + bulkcopy.Close(); } - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + + Helpers.VerifyResults(dstConn, dstTable.Name, 4, 18); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromDatatableAsync.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromDatatableAsync.cs index 7a5eb6d8e7..65531fc03c 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromDatatableAsync.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromDatatableAsync.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -18,69 +19,61 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_AsyncTest3", false); // Use this semaphore to ensure that results are written to the log in the correct order SemaphoreSlim outputSemaphore = new SemaphoreSlim(0, 1); - Task t = TestAsync(srcConstr, dstConstr, dstTable, outputSemaphore); + Task t = TestAsync(srcConstr, dstConstr, outputSemaphore); outputSemaphore.Release(); t.Wait(); Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); } - private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable, SemaphoreSlim outputSemaphore) + private static async Task TestAsync(string srcConstr, string dstConstr, SemaphoreSlim outputSemaphore) { DataSet dataset; SqlDataAdapter adapter; DataTable datatable; using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try - { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) - { - srcConn.Open(); + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_AsyncTest3", "(col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); - dataset = new DataSet("MyDataSet"); - adapter = new SqlDataAdapter(srcCmd); - adapter.Fill(dataset); - datatable = dataset.Tables[0]; + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) + { + srcConn.Open(); - datatable.Rows[0].BeginEdit(); - datatable.Rows[0][0] = 333; - datatable.Rows[0].EndEdit(); + dataset = new DataSet("MyDataSet"); + adapter = new SqlDataAdapter(srcCmd); + adapter.Fill(dataset); + datatable = dataset.Tables[0]; - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - bulkcopy.BatchSize = 7; + datatable.Rows[0].BeginEdit(); + datatable.Rows[0][0] = 333; + datatable.Rows[0].EndEdit(); - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.BatchSize = 7; - ColumnMappings.Add(0, "col1"); - ColumnMappings.Add(1, "col2"); - ColumnMappings.Add(2, "col3"); - bulkcopy.WriteToServer(datatable, DataRowState.Unchanged); - datatable.Rows.GetEnumerator().Reset(); - await bulkcopy.WriteToServerAsync(datatable, DataRowState.Modified); - datatable.Rows.GetEnumerator().Reset(); - await bulkcopy.WriteToServerAsync(datatable, DataRowState.Deleted); - bulkcopy.Close(); - } + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - await outputSemaphore.WaitAsync(); - Helpers.VerifyResults(dstConn, dstTable, 4, 18); + ColumnMappings.Add(0, "col1"); + ColumnMappings.Add(1, "col2"); + ColumnMappings.Add(2, "col3"); + bulkcopy.WriteToServer(datatable, DataRowState.Unchanged); + datatable.Rows.GetEnumerator().Reset(); + await bulkcopy.WriteToServerAsync(datatable, DataRowState.Modified); + datatable.Rows.GetEnumerator().Reset(); + await bulkcopy.WriteToServerAsync(datatable, DataRowState.Deleted); + bulkcopy.Close(); } - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + + await outputSemaphore.WaitAsync(); + Helpers.VerifyResults(dstConn, dstTable.Name, 4, 18); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromReader.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromReader.cs index 9d52892baf..02d101f4e9 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromReader.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromReader.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,45 +17,36 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CopySomeFromReader", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_CopySomeFromReader", "(col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) { - srcConn.Open(); - - using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - bulkcopy.BatchSize = 6; + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.BatchSize = 6; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("EmployeeID", "col1"); - ColumnMappings.Add("BirthDate", "col4"); - ColumnMappings.Add("FirstName", "col2"); - ColumnMappings.Add("LastName", "col3"); + ColumnMappings.Add("EmployeeID", "col1"); + ColumnMappings.Add("BirthDate", "col4"); + ColumnMappings.Add("FirstName", "col2"); + ColumnMappings.Add("LastName", "col3"); - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable, 4, 9); + bulkcopy.WriteToServer(reader); } + Helpers.VerifyResults(dstConn, dstTable.Name, 4, 9); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromRowArray.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromRowArray.cs index be48bf4351..4e6aa12cda 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromRowArray.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromRowArray.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,55 +17,46 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CopySomeFromRowArray", false); DataSet dataset; SqlDataAdapter adapter; DataTable datatable; DataRow[] rows; using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_CopySomeFromRowArray", "(col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) + dataset = new DataSet("MyDataSet"); + adapter = new SqlDataAdapter(srcCmd); + adapter.Fill(dataset); + datatable = dataset.Tables[0]; + rows = new DataRow[datatable.Rows.Count]; + for (int i = 0; i < rows.Length; i++) { - srcConn.Open(); - - dataset = new DataSet("MyDataSet"); - adapter = new SqlDataAdapter(srcCmd); - adapter.Fill(dataset); - datatable = dataset.Tables[0]; - rows = new DataRow[datatable.Rows.Count]; - for (int i = 0; i < rows.Length; i++) - { - rows[i] = datatable.Rows[i]; - } + rows[i] = datatable.Rows[i]; + } - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - bulkcopy.BatchSize = 4; + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.BatchSize = 4; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add(0, "col1"); - ColumnMappings.Add(2, "col3"); + ColumnMappings.Add(0, "col1"); + ColumnMappings.Add(2, "col3"); - bulkcopy.WriteToServer(rows); - bulkcopy.Close(); - } - Helpers.VerifyResults(dstConn, dstTable, 4, 9); + bulkcopy.WriteToServer(rows); + bulkcopy.Close(); } - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + Helpers.VerifyResults(dstConn, dstTable.Name, 4, 9); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromRowArrayAsync.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromRowArrayAsync.cs index b74144fc77..184e109c7f 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromRowArrayAsync.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopySomeFromRowArrayAsync.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -18,17 +19,16 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_AsyncTest2", false); // Use this semaphore to ensure that results are written to the log in the correct order SemaphoreSlim outputSemaphore = new SemaphoreSlim(0, 1); - Task t = TestAsync(srcConstr, dstConstr, dstTable, outputSemaphore); + Task t = TestAsync(srcConstr, dstConstr, outputSemaphore); outputSemaphore.Release(); t.Wait(); Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); } - private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable, SemaphoreSlim outputSemaphore) + private static async Task TestAsync(string srcConstr, string dstConstr, SemaphoreSlim outputSemaphore) { DataSet dataset; SqlDataAdapter adapter; @@ -36,49 +36,41 @@ private static async Task TestAsync(string srcConstr, string dstConstr, string d DataRow[] rows; using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_AsyncTest2", "(col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10), col4 datetime)"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select * from employees", srcConn)) + dataset = new DataSet("MyDataSet"); + adapter = new SqlDataAdapter(srcCmd); + adapter.Fill(dataset); + datatable = dataset.Tables[0]; + rows = new DataRow[datatable.Rows.Count]; + for (int i = 0; i < rows.Length; i++) { - srcConn.Open(); - - dataset = new DataSet("MyDataSet"); - adapter = new SqlDataAdapter(srcCmd); - adapter.Fill(dataset); - datatable = dataset.Tables[0]; - rows = new DataRow[datatable.Rows.Count]; - for (int i = 0; i < rows.Length; i++) - { - rows[i] = datatable.Rows[i]; - } + rows[i] = datatable.Rows[i]; + } - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - bulkcopy.BatchSize = 4; + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.BatchSize = 4; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add(0, "col1"); - ColumnMappings.Add(2, "col3"); + ColumnMappings.Add(0, "col1"); + ColumnMappings.Add(2, "col3"); - await bulkcopy.WriteToServerAsync(rows); - bulkcopy.Close(); - } - await outputSemaphore.WaitAsync(); - Helpers.VerifyResults(dstConn, dstTable, 4, 9); + await bulkcopy.WriteToServerAsync(rows); + bulkcopy.Close(); } - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + await outputSemaphore.WaitAsync(); + Helpers.VerifyResults(dstConn, dstTable.Name, 4, 9); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyVariants.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyVariants.cs index d690f849ca..f0654c57a7 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyVariants.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyVariants.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -15,68 +16,60 @@ public class CopyVariants public void Test() { string constr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Variants", false); - string[] prologue = - { - "create table " + dstTable + "_src (col_1 int primary key, col_2 sql_variant)", - - "insert into " + dstTable + "_src values (0, null)", - "insert into " + dstTable + "_src values (1, convert(int, 0))", - "insert into " + dstTable + "_src values (2, convert(smallint, -32768))", - "insert into " + dstTable + "_src values (3, convert(real, 2.2))", - "insert into " + dstTable + "_src values (4, convert(float, -3303.33303))", - "insert into " + dstTable + "_src values (5, convert(decimal(28,4), 44404.4404))", - "insert into " + dstTable + "_src values (6, convert(money, $555505.5505) )", - "insert into " + dstTable + "_src values (7, convert(smallmoney, $-6.6606) )", - "insert into " + dstTable + "_src values (8, convert(bit, 1) )", - "insert into " + dstTable + "_src values (9, convert(tinyint, 8) )", - "insert into " + dstTable + "_src values (10, convert(uniqueidentifier, '00000000-0000-0000-0000-000000000009') )", - "insert into " + dstTable + "_src values (11, convert(varbinary(756), 0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA0A) )", - "insert into " + dstTable + "_src values (12, convert(varchar(756), '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111101') )", - "insert into " + dstTable + "_src values (13, convert(nvarchar(756), N'???a???????????????????????????????üböuaäZßABCÄboÜOUÖvrhÃã??z?????????????????z?????????A?????a???????????????????????????????üböuaäZßABCÄboÜOUÖvrhÃã??z?????????????????z?????????A?????a???????????????????????????????üböuaäZßABCÄboÜOUÖvrhÃã??z?????????????????z?????????A?????a???????????????????????????????üböuaäZßABCÄboÜOUÖvrhÃã??z?????????????????z?????????A?????a?????') )", - "insert into " + dstTable + "_src values (14, convert(datetime, {ts '2003-01-11 12:54:01.133'}) )", - "insert into " + dstTable + "_src values (15, convert(bigint, 444444444444404) )", - "insert into " + dstTable + "_src values (16, convert(int, -555505) )", - "insert into " + dstTable + "_src values (17, convert(smallint, 16) )", - "insert into " + dstTable + "_src values (18, convert(real, 777707.7) )", - "insert into " + dstTable + "_src values (19, convert(float, -888888808.88018) )", - "insert into " + dstTable + "_src values (20, convert(decimal(28,4), 99999999999999999909.9019) )", - - "create table " + dstTable + "_dst (col_1 int primary key, col_2 sql_variant)", - }; using (SqlConnection dstConn = new SqlConnection(constr)) using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table srcTable = new(dstConn, "SqlBulkCopyTest_Variants_src", "(col_1 int primary key, col_2 sql_variant)"); + using Table dstTable = new(dstConn, "SqlBulkCopyTest_Variants_dst", "(col_1 int primary key, col_2 sql_variant)"); + + string[] prologue = { - foreach (string cmdtext in prologue) - { - Helpers.TryExecute(dstCmd, cmdtext); - } - using (SqlConnection srcConn = new SqlConnection(constr)) - using (SqlCommand srcCmd = new SqlCommand("select * from " + dstTable + "_src", srcConn)) - { - srcConn.Open(); + "insert into " + srcTable.Name + " values (0, null)", + "insert into " + srcTable.Name + " values (1, convert(int, 0))", + "insert into " + srcTable.Name + " values (2, convert(smallint, -32768))", + "insert into " + srcTable.Name + " values (3, convert(real, 2.2))", + "insert into " + srcTable.Name + " values (4, convert(float, -3303.33303))", + "insert into " + srcTable.Name + " values (5, convert(decimal(28,4), 44404.4404))", + "insert into " + srcTable.Name + " values (6, convert(money, $555505.5505) )", + "insert into " + srcTable.Name + " values (7, convert(smallmoney, $-6.6606) )", + "insert into " + srcTable.Name + " values (8, convert(bit, 1) )", + "insert into " + srcTable.Name + " values (9, convert(tinyint, 8) )", + "insert into " + srcTable.Name + " values (10, convert(uniqueidentifier, '00000000-0000-0000-0000-000000000009') )", + "insert into " + srcTable.Name + " values (11, convert(varbinary(756), 0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA0A) )", + "insert into " + srcTable.Name + " values (12, convert(varchar(756), '111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111101') )", + "insert into " + srcTable.Name + " values (13, convert(nvarchar(756), N'???a???????????????????????????????üböuaäZßABCÄboÜOUÖvrhÃã??z?????????????????z?????????A?????a???????????????????????????????üböuaäZßABCÄboÜOUÖvrhÃã??z?????????????????z?????????A?????a???????????????????????????????üböuaäZßABCÄboÜOUÖvrhÃã??z?????????????????z?????????A?????a???????????????????????????????üböuaäZßABCÄboÜOUÖvrhÃã??z?????????????????z?????????A?????a?????') )", + "insert into " + srcTable.Name + " values (14, convert(datetime, {ts '2003-01-11 12:54:01.133'}) )", + "insert into " + srcTable.Name + " values (15, convert(bigint, 444444444444404) )", + "insert into " + srcTable.Name + " values (16, convert(int, -555505) )", + "insert into " + srcTable.Name + " values (17, convert(smallint, 16) )", + "insert into " + srcTable.Name + " values (18, convert(real, 777707.7) )", + "insert into " + srcTable.Name + " values (19, convert(float, -888888808.88018) )", + "insert into " + srcTable.Name + " values (20, convert(decimal(28,4), 99999999999999999909.9019) )", + + }; - using (DbDataReader reader = srcCmd.ExecuteReader()) + foreach (string cmdtext in prologue) + { + Helpers.TryExecute(dstCmd, cmdtext); + } + using (SqlConnection srcConn = new SqlConnection(constr)) + using (SqlCommand srcCmd = new SqlCommand("select * from " + srcTable.Name, srcConn)) + { + srcConn.Open(); + + using (DbDataReader reader = srcCmd.ExecuteReader()) + { + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable + "_dst"; - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, dstTable + "_dst", 2, 21); + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.WriteToServer(reader); } + Helpers.VerifyResults(dstConn, dstTable.Name, 2, 21); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable + "_src"); - Helpers.TryExecute(dstCmd, "drop table " + dstTable + "_dst"); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEvent.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEvent.cs index fedfde00b9..80d51af2e8 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEvent.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEvent.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -24,62 +25,53 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CopyWithEvent", false); DataSet dataset; SqlDataAdapter adapter; DataTable datatable; DataRow[] rows; using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_CopyWithEvent", "(orderid int, customerid nchar(5), rdate datetime, freight money, shipname nvarchar(40))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 100 * from orders", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (orderid int, customerid nchar(5), rdate datetime, freight money, shipname nvarchar(40))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 100 * from orders", srcConn)) + dataset = new DataSet("MyDataSet"); + adapter = new SqlDataAdapter(srcCmd); + adapter.Fill(dataset); + datatable = dataset.Tables[0]; + rows = new DataRow[datatable.Rows.Count]; + for (int i = 0; i < rows.Length; i++) { - srcConn.Open(); - - dataset = new DataSet("MyDataSet"); - adapter = new SqlDataAdapter(srcCmd); - adapter.Fill(dataset); - datatable = dataset.Tables[0]; - rows = new DataRow[datatable.Rows.Count]; - for (int i = 0; i < rows.Length; i++) - { - rows[i] = datatable.Rows[i]; - } + rows[i] = datatable.Rows[i]; } + } - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { - bulkcopy.SqlRowsCopied += new SqlRowsCopiedEventHandler(OnRowCopied); + bulkcopy.SqlRowsCopied += new SqlRowsCopiedEventHandler(OnRowCopied); - bulkcopy.DestinationTableName = dstTable; - bulkcopy.NotifyAfter = 50; + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.NotifyAfter = 50; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add(0, "orderid"); - ColumnMappings.Add(1, "customerid"); - ColumnMappings.Add(4, "rdate"); - ColumnMappings.Add(7, "freight"); - ColumnMappings.Add(8, "shipname"); + ColumnMappings.Add(0, "orderid"); + ColumnMappings.Add(1, "customerid"); + ColumnMappings.Add(4, "rdate"); + ColumnMappings.Add(7, "freight"); + ColumnMappings.Add(8, "shipname"); - bulkcopy.WriteToServer(rows); - bulkcopy.SqlRowsCopied -= new SqlRowsCopiedEventHandler(OnRowCopied); - } - Helpers.VerifyResults(dstConn, dstTable, 5, 100); - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + bulkcopy.WriteToServer(rows); + bulkcopy.SqlRowsCopied -= new SqlRowsCopiedEventHandler(OnRowCopied); } + Helpers.VerifyResults(dstConn, dstTable.Name, 5, 100); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEvent1.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEvent1.cs index 4388594033..63adeb7f41 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEvent1.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEvent1.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -38,48 +39,39 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_CopyWithEvent1", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_CopyWithEvent1", "(orderid int, customerid nchar(5), rdate datetime, freight money, shipname nvarchar(40))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 100 * from orders", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (orderid int, customerid nchar(5), rdate datetime, freight money, shipname nvarchar(40))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 100 * from orders", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, null)) { - srcConn.Open(); - - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, null)) - { - bulkcopy.SqlRowsCopied += new SqlRowsCopiedEventHandler(OnRowCopied); + bulkcopy.SqlRowsCopied += new SqlRowsCopiedEventHandler(OnRowCopied); - bulkcopy.DestinationTableName = dstTable; - bulkcopy.NotifyAfter = 50; + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.NotifyAfter = 50; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("OrderID", "orderid"); - ColumnMappings.Add("CustomerID", "customerid"); - ColumnMappings.Add("RequiredDate", "rdate"); - ColumnMappings.Add("Freight", "freight"); - ColumnMappings.Add("ShipName", "shipname"); + ColumnMappings.Add("OrderID", "orderid"); + ColumnMappings.Add("CustomerID", "customerid"); + ColumnMappings.Add("RequiredDate", "rdate"); + ColumnMappings.Add("Freight", "freight"); + ColumnMappings.Add("ShipName", "shipname"); - bulkcopy.NotifyAfter = 3; - DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader)); - bulkcopy.SqlRowsCopied -= new SqlRowsCopiedEventHandler(OnRowCopied); - bulkcopy.Close(); - } + bulkcopy.NotifyAfter = 3; + DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader)); + bulkcopy.SqlRowsCopied -= new SqlRowsCopiedEventHandler(OnRowCopied); + bulkcopy.Close(); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEventAsync.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEventAsync.cs index f552b0610b..1428aa7683 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEventAsync.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/CopyWithEventAsync.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -26,17 +27,16 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_AsyncTest4", false); // Use this semaphore to ensure that results are written to the log in the correct order SemaphoreSlim outputSemaphore = new SemaphoreSlim(0, 1); - Task t = TestAsync(srcConstr, dstConstr, dstTable, outputSemaphore); + Task t = TestAsync(srcConstr, dstConstr, outputSemaphore); outputSemaphore.Release(); t.Wait(); Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); } - private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable, SemaphoreSlim outputSemaphore) + private static async Task TestAsync(string srcConstr, string dstConstr, SemaphoreSlim outputSemaphore) { DataSet dataset; SqlDataAdapter adapter; @@ -44,56 +44,48 @@ private static async Task TestAsync(string srcConstr, string dstConstr, string d DataRow[] rows; using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_AsyncTest4", "(orderid int, customerid nchar(5), rdate datetime, freight money, shipname nvarchar(40))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 100 * from orders", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (orderid int, customerid nchar(5), rdate datetime, freight money, shipname nvarchar(40))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 100 * from orders", srcConn)) + dataset = new DataSet("MyDataSet"); + adapter = new SqlDataAdapter(srcCmd); + adapter.Fill(dataset); + datatable = dataset.Tables[0]; + rows = new DataRow[datatable.Rows.Count]; + for (int i = 0; i < rows.Length; i++) { - srcConn.Open(); - - dataset = new DataSet("MyDataSet"); - adapter = new SqlDataAdapter(srcCmd); - adapter.Fill(dataset); - datatable = dataset.Tables[0]; - rows = new DataRow[datatable.Rows.Count]; - for (int i = 0; i < rows.Length; i++) - { - rows[i] = datatable.Rows[i]; - } + rows[i] = datatable.Rows[i]; } + } - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { - bulkcopy.SqlRowsCopied += new SqlRowsCopiedEventHandler(OnRowCopied); + bulkcopy.SqlRowsCopied += new SqlRowsCopiedEventHandler(OnRowCopied); - bulkcopy.DestinationTableName = dstTable; - bulkcopy.NotifyAfter = 50; + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.NotifyAfter = 50; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add(0, "orderid"); - ColumnMappings.Add(1, "customerid"); - ColumnMappings.Add(4, "rdate"); - ColumnMappings.Add(7, "freight"); - ColumnMappings.Add(8, "shipname"); + ColumnMappings.Add(0, "orderid"); + ColumnMappings.Add(1, "customerid"); + ColumnMappings.Add(4, "rdate"); + ColumnMappings.Add(7, "freight"); + ColumnMappings.Add(8, "shipname"); - await bulkcopy.WriteToServerAsync(rows); - bulkcopy.SqlRowsCopied -= new SqlRowsCopiedEventHandler(OnRowCopied); - } - await outputSemaphore.WaitAsync(); - Helpers.VerifyResults(dstConn, dstTable, 5, 100); - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); + await bulkcopy.WriteToServerAsync(rows); + bulkcopy.SqlRowsCopied -= new SqlRowsCopiedEventHandler(OnRowCopied); } + await outputSemaphore.WaitAsync(); + Helpers.VerifyResults(dstConn, dstTable.Name, 5, 100); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/DataConversionErrorMessageTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/DataConversionErrorMessageTest.cs index 11c4a262a1..1ffa69e5c4 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/DataConversionErrorMessageTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/DataConversionErrorMessageTest.cs @@ -70,7 +70,9 @@ private void DropTable(SqlConnection sqlConnection, string targetTable) using (var command = new SqlCommand()) { command.Connection = sqlConnection; - command.CommandText = string.Format("DROP TABLE {0}", targetTable); + // NOTE: The drop is guarded so that cleanup is idempotent; the table name embeds a + // GUID, so an unguarded drop that throws would leave it behind forever. + command.CommandText = string.Format("IF (OBJECT_ID('{0}') IS NOT NULL) DROP TABLE {0}", targetTable); command.CommandType = CommandType.Text; command.ExecuteNonQuery(); } @@ -79,10 +81,19 @@ private void DropTable(SqlConnection sqlConnection, string targetTable) public void Dispose() { - DropTable(Connection, TableName); - - Connection.Close(); - Connection.Dispose(); + try + { + DropTable(Connection, TableName); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(InitialDatabase)}: failed to drop '{TableName}': {ex.Message}"); + } + finally + { + Connection.Close(); + Connection.Dispose(); + } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/DestinationTableNameWithSpecialChar.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/DestinationTableNameWithSpecialChar.cs index 230d450f80..1406ae382a 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/DestinationTableNameWithSpecialChar.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/DestinationTableNameWithSpecialChar.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -26,52 +27,36 @@ public void Test() "[dbo]." + "[" + dsttable + "]", // [dbo].[@sometablename] }; - string[] epilogue = - { - "create table " + srctable + "([col1] int)", - "insert into " + srctable + " values (33)", - "create table [" + dsttable + "]([col1] int)", - }; - - string[] prologue = - { - "drop table " + srctable, - "drop table [" + dsttable + "]", - }; - using (SqlConnection dstConn = new SqlConnection(constr)) using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + + using Table srcTableObject = Table.WithName(dstConn, srctable, "([col1] int)"); + using Table dstTableObject = Table.WithName(dstConn, "[" + dsttable + "]", "([col1] int)"); + + Helpers.TryExecute(dstCmd, "insert into " + srctable + " values (33)"); + + using (SqlConnection srcConn = new SqlConnection(constr)) + using (SqlCommand srcCmd = new SqlCommand(string.Format("select * from {0} ", srctable), srcConn)) { - Helpers.ProcessCommandBatch(typeof(SqlConnection), constr, epilogue); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(constr)) - using (SqlCommand srcCmd = new SqlCommand(string.Format("select * from {0} ", srctable), srcConn)) + int expRows = 1; + foreach (string dsttablename in dsttablecombo) { - srcConn.Open(); - - int expRows = 1; - foreach (string dsttablename in dsttablecombo) + using (DbDataReader reader = srcCmd.ExecuteReader()) { - using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dsttablename; - bulkcopy.WriteToServer(reader); - } - Helpers.VerifyResults(dstConn, "[" + dsttable + "]", 1, expRows); + bulkcopy.DestinationTableName = dsttablename; + bulkcopy.WriteToServer(reader); } - expRows++; + Helpers.VerifyResults(dstConn, "[" + dsttable + "]", 1, expRows); } + expRows++; } } - finally - { - Helpers.ProcessCommandBatch(typeof(SqlConnection), constr, prologue); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/ErrorOnRowsMarkedAsDeleted.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/ErrorOnRowsMarkedAsDeleted.cs index 3ac8636467..8d1b2e90fc 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/ErrorOnRowsMarkedAsDeleted.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/ErrorOnRowsMarkedAsDeleted.cs @@ -102,7 +102,7 @@ private static void RunCase(SqlConnection conn, string caseName, SqlBulkCopyInpu } // create SQL table with one int field, similar to the above DataTable - SqlCommand cmd = conn.CreateCommand(); + using SqlCommand cmd = conn.CreateCommand(); cmd.CommandText = "CREATE TABLE [" + tableName + "] (IntVal int)"; cmd.ExecuteNonQuery(); @@ -145,10 +145,18 @@ private static void RunCase(SqlConnection conn, string caseName, SqlBulkCopyInpu } finally { - // delete the table - cmd = conn.CreateCommand(); - cmd.CommandText = "DROP TABLE [" + tableName + "]"; - cmd.ExecuteNonQuery(); + // Best-effort: this runs while a test failure may already be propagating, so a + // failed drop must not surface in its place. The table is named instead. + try + { + using SqlCommand dropCmd = conn.CreateCommand(); + dropCmd.CommandText = "IF (OBJECT_ID('[" + tableName + "]') IS NOT NULL) DROP TABLE [" + tableName + "]"; + dropCmd.ExecuteNonQuery(); + } + catch (Exception ex) + { + Console.WriteLine($"Failed to drop table '{tableName}'; it may be orphaned in the test database. {ex}"); + } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/FireTrigger.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/FireTrigger.cs index 6353197634..bf030ecfff 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/FireTrigger.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/FireTrigger.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,22 +17,6 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_FireTrigger", false); - string dstTable1 = dstTable + "_1"; // this table will receive a value if the trigger fires! - string trigger = dstTable + "_2"; - string[] prologue = - { - "create table " + dstTable + "(col1 int, col2 nvarchar(20), col3 nvarchar(10))", - "create table " + dstTable1 + " (col1 int);", - "create trigger " + trigger + " on " + dstTable + " for INSERT as insert into " + dstTable1 + " values (333)" - }; - string[] epilogue = - { - "drop table " + dstTable1 + " ", - "drop trigger " + trigger + " ", - "drop table " + dstTable - }; - string sourceTable = "employees"; string sourceQueryTemplate = "select top 5 EmployeeID, LastName, FirstName from {0}"; string sourceQuery = string.Format(sourceQueryTemplate, sourceTable); @@ -40,41 +25,43 @@ public void Test() using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - Helpers.ProcessCommandBatch(dstCmd, prologue); - try + // Dropping a table also drops the triggers defined on it, so the trigger below + // needs no separate cleanup. dstTable is declared last so that it - and its + // trigger - are dropped before the table the trigger writes into. + using Table dstTable1 = new(dstConn, "SqlBulkCopyTest_FireTrigger_1", "(col1 int)"); + using Table dstTable = new(dstConn, "SqlBulkCopyTest_FireTrigger", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + Helpers.TryExecute(dstCmd, + "create trigger " + DataTestUtility.GetShortName("SqlBulkCopyTest_FireTrigger_2", false) + + " on " + dstTable.Name + " for INSERT as insert into " + dstTable1.Name + " values (333)"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) { - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand(sourceQuery, srcConn)) + srcConn.Open(); + + using (DbDataReader reader = srcCmd.ExecuteReader()) { - srcConn.Open(); + SqlBulkCopyOptions option = SqlBulkCopyOptions.FireTriggers; - using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, option, null)) { - SqlBulkCopyOptions option = SqlBulkCopyOptions.FireTriggers; - - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, option, null)) - { - bulkcopy.DestinationTableName = dstTable; - bulkcopy.WriteToServer(reader); - } + bulkcopy.DestinationTableName = dstTable.Name; + bulkcopy.WriteToServer(reader); } + } - dstCmd.CommandText = "select top 2 * from " + dstTable1 + " "; - using (DbDataReader reader2 = dstCmd.ExecuteReader()) - { - Assert.True(reader2.Read(), "Failed to read!"); + dstCmd.CommandText = "select top 2 * from " + dstTable1.Name; + using (DbDataReader reader2 = dstCmd.ExecuteReader()) + { + Assert.True(reader2.Read(), "Failed to read!"); - Assert.True(reader2[0] is int, "Unexpected Field(0) type: " + reader2[0].GetType()); + Assert.True(reader2[0] is int, "Unexpected Field(0) type: " + reader2[0].GetType()); - Assert.True((int)(reader2[0]) == 333, "Unexpected Field(0) value: " + reader2[0]); - } + Assert.True((int)(reader2[0]) == 333, "Unexpected Field(0) value: " + reader2[0]); } } - finally - { - Helpers.ProcessCommandBatch(dstCmd, epilogue); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Helpers.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Helpers.cs index 274750bc09..0a447da8b8 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Helpers.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Helpers.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -10,40 +10,6 @@ namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy { public class Helpers { - internal static void ProcessCommandBatch(Type connType, string constr, string[] batch) - { - if (batch.Length > 0) - { - object[] activatorArgs = new object[1]; - activatorArgs[0] = constr; - using (DbConnection conn = (DbConnection)Activator.CreateInstance(connType, activatorArgs)) - { - conn.Open(); - DbCommand cmd = conn.CreateCommand(); - - ProcessCommandBatch(cmd, batch); - } - } - } - - internal static void ProcessCommandBatch(DbCommand cmd, string[] batch) - { - foreach (string cmdtext in batch) - { - Helpers.TryExecute(cmd, cmdtext); - } - } - - public static int TryDropTable(string dstConstr, string tableName) - { - using (SqlConnection dropConn = new SqlConnection(dstConstr)) - using (SqlCommand dropCmd = dropConn.CreateCommand()) - { - dropConn.Open(); - return Helpers.TryExecute(dropCmd, "drop table " + tableName); - } - } - public static int TryExecute(DbCommand cmd, string strText) { cmd.CommandText = strText; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/HiddenTargetColumn.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/HiddenTargetColumn.cs index 0f18df215e..08c7204127 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/HiddenTargetColumn.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/HiddenTargetColumn.cs @@ -4,6 +4,7 @@ using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -15,17 +16,13 @@ public class HiddenTargetColumn public void WriteToServer_CopyToHiddenTargetColumn_ThrowsSqlException() { string connectionString = DataTestUtility.TCPConnectionString; - string destinationTable = DataTestUtility.GetShortName("HiddenTargetColumn"); - string destinationHistoryTable = DataTestUtility.GetShortName("HiddenTargetColumn_History"); using (SqlConnection dstConn = new SqlConnection(connectionString)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try - { - DataTestUtility.CreateTable(dstConn, destinationTable, $""" + // Drops the history table as well as the table itself, in the required order. + using TemporalTable destinationTable = new(dstConn, "HiddenTargetColumn", "HiddenTargetColumn_History", """ ( Column1 int primary key not null, Column2 nvarchar(10) not null, @@ -34,42 +31,31 @@ Column2 nvarchar(10) not null, ValidTo datetime2 generated always as row end hidden not null, period for system_time (ValidFrom, ValidTo) ) -with (system_versioning = on(history_table = dbo.{destinationHistoryTable})); """); - using (SqlConnection srcConn = new SqlConnection(connectionString)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, FirstName, LastName, HireDate, sysdatetime() as CurrentDate from employees", srcConn)) - { - srcConn.Open(); + using (SqlConnection srcConn = new SqlConnection(connectionString)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, FirstName, LastName, HireDate, sysdatetime() as CurrentDate from employees", srcConn)) + { + srcConn.Open(); - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = destinationTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) + { + bulkcopy.DestinationTableName = destinationTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("EmployeeID", "Column1"); - ColumnMappings.Add("LastName", "Column2"); - ColumnMappings.Add("FirstName", "Employee's First Name"); - ColumnMappings.Add("HireDate", "ValidFrom"); - ColumnMappings.Add("CurrentDate", "ValidTo"); + ColumnMappings.Add("EmployeeID", "Column1"); + ColumnMappings.Add("LastName", "Column2"); + ColumnMappings.Add("FirstName", "Employee's First Name"); + ColumnMappings.Add("HireDate", "ValidFrom"); + ColumnMappings.Add("CurrentDate", "ValidTo"); - SqlException sqlEx = Assert.Throws(() => bulkcopy.WriteToServer(reader)); + SqlException sqlEx = Assert.Throws(() => bulkcopy.WriteToServer(reader)); - Assert.Equal(13536, sqlEx.Number); - Assert.StartsWith("Cannot insert an explicit value into a GENERATED ALWAYS column in table", sqlEx.Message); - } + Assert.Equal(13536, sqlEx.Number); + Assert.StartsWith("Cannot insert an explicit value into a GENERATED ALWAYS column in table", sqlEx.Message); } } - finally - { - DataTestUtility.RunNonQuery(connectionString, $""" -alter table {destinationTable} set (system_versioning = off); -alter table {destinationTable} drop period for system_time; -"""); - DataTestUtility.DropTable(dstConn, destinationTable); - DataTestUtility.DropTable(dstConn, destinationHistoryTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/InvalidAccessFromEvent.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/InvalidAccessFromEvent.cs index 19bdf2193e..12411f52b5 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/InvalidAccessFromEvent.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/InvalidAccessFromEvent.cs @@ -1,10 +1,11 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -54,9 +55,7 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_InvalidAccessFromEvent", false); _dstConstr = dstConstr; - _dstTable = dstTable; DataSet dataset; SqlDataAdapter adapter; @@ -77,30 +76,26 @@ public void Test() _dstConn.Open(); _dstcmd = _dstConn.CreateCommand(); - try - { - Helpers.TryExecute(_dstcmd, "create table " + dstTable + " (orderid int, customerid nchar(5), rdate datetime, freight money, shipname nvarchar(40))"); - _dstcmd.CommandText = "truncate table " + dstTable; - - expectedErrorMsg = SystemDataResourceManager.Instance.SQL_ConnectionLockedForBcpEvent; - InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedRollback)); - InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedCommit)); - InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedChangeDatabase)); - InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedExecute)); - InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedBulkCopy)); - - // this will close the connect which is valid so it must be the last test! - expectedErrorMsg = string.Format( - SystemDataResourceManager.Instance.ADP_OpenConnectionRequired, - "WriteToServer", - SystemDataResourceManager.Instance.ADP_ConnectionStateMsg_Closed); - InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedClose)); - } - finally - { - // the original connection is probably trashed - Helpers.TryDropTable(dstConstr, dstTable); - } + // Table disposal falls back to a fresh connection, which matters here because the + // test deliberately leaves the original connection unusable. + using Table dstTable = new(_dstConn, "SqlBulkCopyTest_InvalidAccessFromEvent", + "(orderid int, customerid nchar(5), rdate datetime, freight money, shipname nvarchar(40))"); + _dstTable = dstTable.Name; + _dstcmd.CommandText = "truncate table " + dstTable.Name; + + expectedErrorMsg = SystemDataResourceManager.Instance.SQL_ConnectionLockedForBcpEvent; + InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedRollback)); + InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedCommit)); + InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedChangeDatabase)); + InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedExecute)); + InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedBulkCopy)); + + // this will close the connect which is valid so it must be the last test! + expectedErrorMsg = string.Format( + SystemDataResourceManager.Instance.ADP_OpenConnectionRequired, + "WriteToServer", + SystemDataResourceManager.Instance.ADP_ConnectionStateMsg_Closed); + InnerTest(new SqlRowsCopiedEventHandler(OnRowCopiedClose)); } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/KeepNulls.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/KeepNulls.cs index 5228c1a35d..4e0c663610 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/KeepNulls.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/KeepNulls.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,42 +17,33 @@ public void Test() { string srcconstr = DataTestUtility.TCPConnectionString; string dstconstr = DataTestUtility.TCPConnectionString; - string srctable = DataTestUtility.GetShortName("SqlBulkCopyTest_KeepNulls0", false); - string dsttable = DataTestUtility.GetShortName("SqlBulkCopyTest_KeepNulls1", false); using SqlConnection destConn = new(dstconstr); destConn.Open(); using SqlCommand dstcmd = destConn.CreateCommand(); - Helpers.TryExecute(dstcmd, "create table " + srctable + " (col1 int, col2 text, col3 text)"); - Helpers.TryExecute(dstcmd, "insert into " + srctable + "(col1, col3) values (1, 'Michael')"); - Helpers.TryExecute(dstcmd, "insert into " + srctable + "(col1, col2, col3) values (2, 'Quark', 'Astrid')"); - Helpers.TryExecute(dstcmd, "insert into " + srctable + "(col1, col2) values (66, 'K�se');"); - Helpers.TryExecute(dstcmd, "create table " + dsttable + " (col1 int identity(1,1), col2 text default 'Jogurt', col3 text)"); + using Table srctable = new(destConn, "SqlBulkCopyTest_KeepNulls0", "(col1 int, col2 text, col3 text)"); + using Table dsttable = new(destConn, "SqlBulkCopyTest_KeepNulls1", "(col1 int identity(1,1), col2 text default 'Jogurt', col3 text)"); + + Helpers.TryExecute(dstcmd, "insert into " + srctable.Name + "(col1, col3) values (1, 'Michael')"); + Helpers.TryExecute(dstcmd, "insert into " + srctable.Name + "(col1, col2, col3) values (2, 'Quark', 'Astrid')"); + Helpers.TryExecute(dstcmd, "insert into " + srctable.Name + "(col1, col2) values (66, 'K�se');"); using SqlConnection sourceConn = new(srcconstr); sourceConn.Open(); - using SqlCommand srccmd = new("select * from " + srctable, sourceConn); + using SqlCommand srccmd = new("select * from " + srctable.Name, sourceConn); using IDataReader reader = srccmd.ExecuteReader(); - try - { - using SqlBulkCopy bulkcopy = new(destConn, SqlBulkCopyOptions.KeepNulls, null); - bulkcopy.DestinationTableName = dsttable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("col1", "col1"); - ColumnMappings.Add("col2", "col2"); - ColumnMappings.Add("col3", "col3"); - - bulkcopy.WriteToServer(reader); - Helpers.VerifyResults(destConn, dsttable, 3, 3); - } - finally - { - Helpers.TryDropTable(dstconstr, srctable); - Helpers.TryDropTable(dstconstr, dsttable); - } + using SqlBulkCopy bulkcopy = new(destConn, SqlBulkCopyOptions.KeepNulls, null); + bulkcopy.DestinationTableName = dsttable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + ColumnMappings.Add("col1", "col1"); + ColumnMappings.Add("col2", "col2"); + ColumnMappings.Add("col3", "col3"); + + bulkcopy.WriteToServer(reader); + Helpers.VerifyResults(destConn, dsttable.Name, 3, 3); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetColumn.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetColumn.cs index ac5651f34f..85c0a4431b 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetColumn.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetColumn.cs @@ -1,10 +1,11 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -17,42 +18,33 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_MissingTargetColumn", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_MissingTargetColumn", "(col1 int, col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col3 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - srcConn.Open(); - - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + bulkcopy.DestinationTableName = dstTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("EmployeeID", "col1"); - ColumnMappings.Add("LastName", "col2"); // this column does not exist - ColumnMappings.Add("FirstName", "col3"); + ColumnMappings.Add("EmployeeID", "col1"); + ColumnMappings.Add("LastName", "col2"); // this column does not exist + ColumnMappings.Add("FirstName", "col3"); - string errorMsg = SystemDataResourceManager.Instance.SQL_BulkLoadNonMatchingColumnName; - errorMsg = string.Format(errorMsg, "col2"); + string errorMsg = SystemDataResourceManager.Instance.SQL_BulkLoadNonMatchingColumnName; + errorMsg = string.Format(errorMsg, "col2"); - DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader), exceptionMessage: errorMsg); - } + DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader), exceptionMessage: errorMsg); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetColumns.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetColumns.cs index 2728b1efb4..d1f035f1bd 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetColumns.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetColumns.cs @@ -1,10 +1,11 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -17,42 +18,33 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_MissingTargetColumns", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_MissingTargetColumns", "(col1 int, col2 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - srcConn.Open(); - - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + bulkcopy.DestinationTableName = dstTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("EmployeeID", "col1"); - ColumnMappings.Add("LastName", "col3"); // this column does not exist - ColumnMappings.Add("FirstName", "col4"); // this column does not exist + ColumnMappings.Add("EmployeeID", "col1"); + ColumnMappings.Add("LastName", "col3"); // this column does not exist + ColumnMappings.Add("FirstName", "col4"); // this column does not exist - string errorMsg = SystemDataResourceManager.Instance.SQL_BulkLoadNonMatchingColumnName; - errorMsg = string.Format(errorMsg, "col3,col4"); + string errorMsg = SystemDataResourceManager.Instance.SQL_BulkLoadNonMatchingColumnName; + errorMsg = string.Format(errorMsg, "col3,col4"); - DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader), exceptionMessage: errorMsg); - } + DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader), exceptionMessage: errorMsg); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetTable.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetTable.cs index af103c1c0a..a8d255224e 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetTable.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/MissingTargetTable.cs @@ -19,7 +19,6 @@ public void Test() string dstConstr = DataTestUtility.TCPConnectionString; string targetTable = DataTestUtility.GetShortName("@SqlBulkCopyTest_MissingTargetTable", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/OrderHintTransaction.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/OrderHintTransaction.cs index e8a0571467..3e8b70facc 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/OrderHintTransaction.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/OrderHintTransaction.cs @@ -27,7 +27,7 @@ public void Test() using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - SqlTransaction txn = dstConn.BeginTransaction(); + using SqlTransaction txn = dstConn.BeginTransaction(); dstCmd.Transaction = txn; Helpers.TryExecute(dstCmd, initialQuery); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/SpecialCharacterNames.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/SpecialCharacterNames.cs index 1c10feebad..f1d82e71c1 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/SpecialCharacterNames.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/SpecialCharacterNames.cs @@ -1,8 +1,9 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -28,36 +29,30 @@ public void Test() dstTable = EscapeIdentifier(dstTable); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + // The table is not currently created inside the schema, but it is declared second so + // that it is dropped first regardless: a schema cannot be dropped while it still + // contains objects. + using Schema schema = Schema.WithName(dstConn, dstschema); + using Table table = Table.WithName(dstConn, dstTable, "(orderid int, customerid nchar(5))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 2 orderid, customerid from orders", srcConn)) { - Helpers.TryExecute(dstCmd, "create schema " + dstschema); - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (orderid int, customerid nchar(5))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 2 orderid, customerid from orders", srcConn)) + using (SqlDataReader srcreader = srcCmd.ExecuteReader()) { - srcConn.Open(); - - using (SqlDataReader srcreader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) { - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn)) - { - bulkcopy.DestinationTableName = dstTable; + bulkcopy.DestinationTableName = dstTable; - bulkcopy.WriteToServer(srcreader); - } + bulkcopy.WriteToServer(srcreader); } - Helpers.VerifyResults(dstConn, dstTable, 2, 2); } - } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - Helpers.TryExecute(dstCmd, "drop schema " + dstschema); + Helpers.VerifyResults(dstConn, dstTable, 2, 2); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TableLock.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TableLock.cs index 69c090d655..f542edc215 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TableLock.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TableLock.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,41 +17,33 @@ public void Test() { string srcconstr = DataTestUtility.TCPConnectionString; string dstconstr = DataTestUtility.TCPConnectionString; - string srctable = DataTestUtility.GetShortName("SqlBulkCopyTest_TableLock0", false); - string dsttable = DataTestUtility.GetShortName("SqlBulkCopyTest_TableLock1", false); using SqlConnection destConn = new(dstconstr); destConn.Open(); using SqlCommand dstcmd = destConn.CreateCommand(); - Helpers.TryExecute(dstcmd, "create table " + srctable + " (col1 int, col2 text, col3 text)"); - Helpers.TryExecute(dstcmd, "insert into " + srctable + "(col1, col3) values (1, 'Michael')"); - Helpers.TryExecute(dstcmd, "insert into " + srctable + "(col1, col2, col3) values (2, 'Quark', 'Astrid')"); - Helpers.TryExecute(dstcmd, "insert into " + srctable + "(col1, col2) values (66, 'K�se');"); - Helpers.TryExecute(dstcmd, "create table " + dsttable + " (col1 int identity(1,1), col2 text default 'Jogurt', col3 text)"); + using Table srctable = new(destConn, "SqlBulkCopyTest_TableLock0", "(col1 int, col2 text, col3 text)"); + using Table dsttable = new(destConn, "SqlBulkCopyTest_TableLock1", "(col1 int identity(1,1), col2 text default 'Jogurt', col3 text)"); + + Helpers.TryExecute(dstcmd, "insert into " + srctable.Name + "(col1, col3) values (1, 'Michael')"); + Helpers.TryExecute(dstcmd, "insert into " + srctable.Name + "(col1, col2, col3) values (2, 'Quark', 'Astrid')"); + Helpers.TryExecute(dstcmd, "insert into " + srctable.Name + "(col1, col2) values (66, 'K�se');"); using SqlConnection sourceConn = new(srcconstr); sourceConn.Open(); - using SqlCommand srccmd = new SqlCommand("select * from " + srctable, sourceConn); + using SqlCommand srccmd = new SqlCommand("select * from " + srctable.Name, sourceConn); using IDataReader reader = srccmd.ExecuteReader(); - try - { - using SqlBulkCopy bulkcopy = new(destConn, SqlBulkCopyOptions.TableLock, null); - bulkcopy.DestinationTableName = dsttable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - ColumnMappings.Add("col1", "col1"); - ColumnMappings.Add("col2", "col2"); - ColumnMappings.Add("col3", "col3"); - - bulkcopy.WriteToServer(reader); - Helpers.VerifyResults(destConn, dsttable, 3, 3); - } - finally - { - Helpers.TryDropTable(dstconstr, srctable); - Helpers.TryDropTable(dstconstr, dsttable); - } + + using SqlBulkCopy bulkcopy = new(destConn, SqlBulkCopyOptions.TableLock, null); + bulkcopy.DestinationTableName = dsttable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + ColumnMappings.Add("col1", "col1"); + ColumnMappings.Add("col2", "col2"); + ColumnMappings.Add("col3", "col3"); + + bulkcopy.WriteToServer(reader); + Helpers.VerifyResults(destConn, dsttable.Name, 3, 3); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TestBulkCopyWithUTF8.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TestBulkCopyWithUTF8.cs index 21cf670ae7..daa63c7d03 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TestBulkCopyWithUTF8.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TestBulkCopyWithUTF8.cs @@ -4,6 +4,7 @@ using System; using System.Data; +using System.Text; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; using Xunit; @@ -17,11 +18,10 @@ namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy [Trait("Set", "2")] public sealed class TestBulkCopyWithUtf8 : IDisposable { - private static string s_sourceTable = DataTestUtility.GetShortName("SourceTableForUTF8Data"); - private static string s_destinationTable = DataTestUtility.GetShortName("DestinationTableForUTF8Data"); - private static string s_testValue = "test"; - private static byte[] s_testValueInUtf8Bytes = new byte[] { 0x74, 0x65, 0x73, 0x74 }; - private static readonly string s_insertQuery = $"INSERT INTO {s_sourceTable} VALUES('{s_testValue}')"; + private static readonly string s_sourceTable = DataTestUtility.GetShortName("SourceTableForUTF8Data"); + private static readonly string s_destinationTable = DataTestUtility.GetShortName("DestinationTableForUTF8Data"); + private static readonly string s_testValue = GlobalizationTestData.CreatePacketSpanningText(); + private static readonly byte[] s_testValueInUtf8Bytes = Encoding.UTF8.GetBytes(s_testValue); /// /// Constructor: Initializes and populates source and destination tables required for the tests. @@ -37,7 +37,7 @@ public TestBulkCopyWithUtf8() using SqlConnection sourceConnection = new SqlConnection(GetConnectionString(true)); sourceConnection.Open(); - SetupTables(sourceConnection, s_sourceTable, s_destinationTable, s_insertQuery); + SetupTables(sourceConnection, s_sourceTable, s_destinationTable); } /// @@ -61,11 +61,14 @@ public void Dispose() /// /// Builds a connection string with or without Multiple Active Result Sets (MARS) property. /// + /// Whether Multiple Active Result Sets is enabled. + /// A connection string configured with a small packet size for boundary coverage. private string GetConnectionString(bool enableMars) { return new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString) { - MultipleActiveResultSets = enableMars + MultipleActiveResultSets = enableMars, + PacketSize = 512 }.ConnectionString; } @@ -73,14 +76,18 @@ private string GetConnectionString(bool enableMars) /// Creates source and destination tables with a varchar(max) column with a collation setting /// that stores the data in UTF8 encoding and inserts the data in the source table. /// - private void SetupTables(SqlConnection connection, string sourceTable, string destinationTable, string insertQuery) + /// The open connection used to create and populate the tables. + /// The source table name. + /// The destination table name. + private void SetupTables(SqlConnection connection, string sourceTable, string destinationTable) { string columnDefinition = "(str_col varchar(max) COLLATE Latin1_General_100_CS_AS_KS_WS_SC_UTF8)"; DataTestUtility.CreateTable(connection, sourceTable, columnDefinition); DataTestUtility.CreateTable(connection, destinationTable, columnDefinition); using SqlCommand insertCommand = connection.CreateCommand(); - insertCommand.CommandText = insertQuery; - Helpers.TryExecute(insertCommand, insertQuery); + insertCommand.CommandText = $"INSERT INTO {sourceTable} VALUES(@value)"; + insertCommand.Parameters.Add(new SqlParameter("@value", SqlDbType.NVarChar, -1) { Value = s_testValue }); + insertCommand.ExecuteNonQuery(); } /// @@ -105,6 +112,11 @@ public void BulkCopy_Utf8Data_ShouldMatchSource(bool isMarsEnabled, bool enableS using SqlConnection destinationConnection = new SqlConnection(connectionString); destinationConnection.Open(); + using (SqlCommand sourceVerifyCommand = new SqlCommand($"SELECT CAST(str_col AS varbinary(max)) FROM {s_sourceTable}", sourceConnection)) + { + Assert.Equal(s_testValueInUtf8Bytes, sourceVerifyCommand.ExecuteScalar()); + } + // Read data from source table using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection); using SqlDataReader reader = sourceDataCommand.ExecuteReader(CommandBehavior.SequentialAccess); @@ -135,7 +147,7 @@ public void BulkCopy_Utf8Data_ShouldMatchSource(bool isMarsEnabled, bool enableS Assert.Equal(1, Convert.ToInt16(countCommand.ExecuteScalar())); // Read the data from destination table as varbinary to verify the UTF-8 byte sequence - using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection); + using SqlCommand verifyCommand = new SqlCommand($"SELECT CAST(str_col AS varbinary(max)) FROM {s_destinationTable}", destinationConnection); using SqlDataReader verifyReader = verifyCommand.ExecuteReader(CommandBehavior.SequentialAccess); // Verify that we have data in the destination table @@ -170,6 +182,11 @@ public async Task BulkCopy_Utf8Data_ShouldMatchSource_Async(bool isMarsEnabled, using SqlConnection destinationConnection = new SqlConnection(connectionString); await destinationConnection.OpenAsync(); + using (SqlCommand sourceVerifyCommand = new SqlCommand($"SELECT CAST(str_col AS varbinary(max)) FROM {s_sourceTable}", sourceConnection)) + { + Assert.Equal(s_testValueInUtf8Bytes, await sourceVerifyCommand.ExecuteScalarAsync()); + } + // Read data from source table using SqlCommand sourceDataCommand = new SqlCommand($"SELECT str_col FROM {s_sourceTable}", sourceConnection); using SqlDataReader reader = await sourceDataCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); @@ -200,7 +217,7 @@ public async Task BulkCopy_Utf8Data_ShouldMatchSource_Async(bool isMarsEnabled, Assert.Equal(1, Convert.ToInt16(await countCommand.ExecuteScalarAsync())); // Read the data from destination table as varbinary to verify the UTF-8 byte sequence - using SqlCommand verifyCommand = new SqlCommand($"SELECT cast(str_col as varbinary) FROM {s_destinationTable}", destinationConnection); + using SqlCommand verifyCommand = new SqlCommand($"SELECT CAST(str_col AS varbinary(max)) FROM {s_destinationTable}", destinationConnection); using SqlDataReader verifyReader = await verifyCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); // Verify that we have data in the destination table diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction.cs index b5f5e32b53..1df72a8dee 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction.cs @@ -1,10 +1,11 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -17,43 +18,34 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Transaction0", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_Transaction0", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, null)) { - srcConn.Open(); + bulkcopy.DestinationTableName = dstTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, null)) + using SqlTransaction myTrans = dstConn.BeginTransaction(); + try { - bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - - SqlTransaction myTrans = dstConn.BeginTransaction(); - try - { - DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader)); - } - finally - { - myTrans.Rollback(); - } + DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader)); + } + finally + { + myTrans.Rollback(); } } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction1.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction1.cs index 665b5a5108..863be57734 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction1.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction1.cs @@ -1,10 +1,11 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -17,47 +18,38 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Transaction1", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_Transaction1", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, null)) { - srcConn.Open(); + bulkcopy.DestinationTableName = dstTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, null)) - { - bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + using SqlCommand myCmd = dstConn.CreateCommand(); + myCmd.CommandText = "begin transaction"; + myCmd.ExecuteNonQuery(); - SqlCommand myCmd = dstConn.CreateCommand(); - myCmd.CommandText = "begin transaction"; + try + { + DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader)); + } + finally + { + myCmd.CommandText = "rollback transaction"; myCmd.ExecuteNonQuery(); - - try - { - DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader)); - } - finally - { - myCmd.CommandText = "rollback transaction"; - myCmd.ExecuteNonQuery(); - } } } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction2.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction2.cs index 9c3f69e28a..a99ff3607a 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction2.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction2.cs @@ -1,9 +1,10 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -16,54 +17,45 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Transaction2", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_Transaction2", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) { - srcConn.Open(); - - using (DbDataReader reader = srcCmd.ExecuteReader()) + using SqlTransaction myTrans = dstConn.BeginTransaction(); + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.Default, myTrans)) { - SqlTransaction myTrans = dstConn.BeginTransaction(); - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.Default, myTrans)) - { - bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + bulkcopy.DestinationTableName = dstTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - try - { - bulkcopy.WriteToServer(reader); - SqlCommand myCmd = dstConn.CreateCommand(); - myCmd.CommandText = "select * from " + dstTable; - myCmd.Transaction = myTrans; - using (DbDataReader reader1 = myCmd.ExecuteReader()) - { - Assert.True(reader1.HasRows, "Expected reader to have rows."); - } - } - finally + try + { + bulkcopy.WriteToServer(reader); + using SqlCommand myCmd = dstConn.CreateCommand(); + myCmd.CommandText = "select * from " + dstTable.Name; + myCmd.Transaction = myTrans; + using (DbDataReader reader1 = myCmd.ExecuteReader()) { - myTrans.Rollback(); + Assert.True(reader1.HasRows, "Expected reader to have rows."); } } - - Helpers.CheckTableRows(dstConn, dstTable, false); + finally + { + myTrans.Rollback(); + } } + + Helpers.CheckTableRows(dstConn, dstTable.Name, false); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction3.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction3.cs index f28aaad8c3..4735e7aa3c 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction3.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction3.cs @@ -1,10 +1,11 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -17,48 +18,39 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Transaction3", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_Transaction3", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlConnection conn3 = new SqlConnection(srcConstr)) { - srcConn.Open(); - - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlConnection conn3 = new SqlConnection(srcConstr)) + conn3.Open(); + // Start a local transaction on the wrong connection. + using SqlTransaction myTrans = conn3.BeginTransaction(); + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.Default, myTrans)) { - conn3.Open(); - // Start a local transaction on the wrong connection. - SqlTransaction myTrans = conn3.BeginTransaction(); - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.Default, myTrans)) - { - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - bulkcopy.DestinationTableName = dstTable; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; + bulkcopy.DestinationTableName = dstTable.Name; - string exceptionMsg = SystemDataResourceManager.Instance.ADP_TransactionConnectionMismatch; - DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader), exceptionMessage: exceptionMsg); + string exceptionMsg = SystemDataResourceManager.Instance.ADP_TransactionConnectionMismatch; + DataTestUtility.AssertThrows(() => bulkcopy.WriteToServer(reader), exceptionMessage: exceptionMsg); - SqlCommand myCmd = dstConn.CreateCommand(); - myCmd.CommandText = "select * from " + dstTable; - myCmd.Transaction = myTrans; + using SqlCommand myCmd = dstConn.CreateCommand(); + myCmd.CommandText = "select * from " + dstTable.Name; + myCmd.Transaction = myTrans; - DataTestUtility.AssertThrows(() => myCmd.ExecuteReader(), exceptionMessage: exceptionMsg); - } + DataTestUtility.AssertThrows(() => myCmd.ExecuteReader(), exceptionMessage: exceptionMsg); } } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction4.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction4.cs index 2c48368483..7e86f8b640 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction4.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/Transaction4.cs @@ -1,10 +1,11 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Data.Common; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -17,35 +18,26 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_Transaction4", false); using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_Transaction4", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlConnection conn3 = new SqlConnection(srcConstr)) { - srcConn.Open(); - - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlConnection conn3 = new SqlConnection(srcConstr)) - { - conn3.Open(); - // Start a local transaction on the wrong connection. - SqlTransaction myTrans = conn3.BeginTransaction(); - string errorMsg = SystemDataResourceManager.Instance.SQL_BulkLoadConflictingTransactionOption; - DataTestUtility.AssertThrows(() => new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, myTrans), exceptionMessage: errorMsg); - } + conn3.Open(); + // Start a local transaction on the wrong connection. + using SqlTransaction myTrans = conn3.BeginTransaction(); + string errorMsg = SystemDataResourceManager.Instance.SQL_BulkLoadConflictingTransactionOption; + DataTestUtility.AssertThrows(() => new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, myTrans), exceptionMessage: errorMsg); } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TransactionTestAsync.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TransactionTestAsync.cs index 537f68d8f0..76ce1f9ffd 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TransactionTestAsync.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/TransactionTestAsync.cs @@ -1,4 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. @@ -6,6 +6,7 @@ using System.Data.Common; using System.Threading.Tasks; using Microsoft.Data.SqlClient.ManualTesting.Tests; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; using Xunit; namespace Microsoft.Data.SqlClient.ManualTests.BulkCopy @@ -18,49 +19,40 @@ public void Test() { string srcConstr = DataTestUtility.TCPConnectionString; string dstConstr = DataTestUtility.TCPConnectionString; - string dstTable = DataTestUtility.GetShortName("SqlBulkCopyTest_TransactionTestAsync", false); - Task t = TestAsync(srcConstr, dstConstr, dstTable); + Task t = TestAsync(srcConstr, dstConstr); DataTestUtility.AssertThrowsInner(() => t.Wait()); Assert.True(t.IsCompleted, "Task did not complete! Status: " + t.Status); } - private static async Task TestAsync(string srcConstr, string dstConstr, string dstTable) + private static async Task TestAsync(string srcConstr, string dstConstr) { using (SqlConnection dstConn = new SqlConnection(dstConstr)) - using (SqlCommand dstCmd = dstConn.CreateCommand()) { dstConn.Open(); - try + using Table dstTable = new Table(dstConn, "SqlBulkCopyTest_TransactionTestAsync", "(col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + + using (SqlConnection srcConn = new SqlConnection(srcConstr)) + using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) { - Helpers.TryExecute(dstCmd, "create table " + dstTable + " (col1 int, col2 nvarchar(20), col3 nvarchar(10))"); + srcConn.Open(); - using (SqlConnection srcConn = new SqlConnection(srcConstr)) - using (SqlCommand srcCmd = new SqlCommand("select top 5 EmployeeID, LastName, FirstName from employees", srcConn)) + using (DbDataReader reader = srcCmd.ExecuteReader()) + using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, null)) { - srcConn.Open(); + bulkcopy.DestinationTableName = dstTable.Name; + SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - using (DbDataReader reader = srcCmd.ExecuteReader()) - using (SqlBulkCopy bulkcopy = new SqlBulkCopy(dstConn, SqlBulkCopyOptions.UseInternalTransaction, null)) + using SqlTransaction myTrans = dstConn.BeginTransaction(); + try { - bulkcopy.DestinationTableName = dstTable; - SqlBulkCopyColumnMappingCollection ColumnMappings = bulkcopy.ColumnMappings; - - SqlTransaction myTrans = dstConn.BeginTransaction(); - try - { - await bulkcopy.WriteToServerAsync(reader); - } - finally - { - myTrans.Rollback(); - } + await bulkcopy.WriteToServerAsync(reader); + } + finally + { + myTrans.Rollback(); } } } - finally - { - Helpers.TryExecute(dstCmd, "drop table " + dstTable); - } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/UnprivilegedLogin.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/UnprivilegedLogin.cs index 7fd5b36237..f78ea7334d 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/UnprivilegedLogin.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/BulkCopy/UnprivilegedLogin.cs @@ -52,19 +52,33 @@ public UnprivilegedLogin() // the actual tests. The user associated with the latter connection will be denied SELECT permissions // over master.sys.all_columns. _managementConnection = new SqlConnection(DataTestUtility.TCPConnectionString); - _managementConnection.Open(); - _unprivilegedLogin = new ServerLogin(_managementConnection, nameof(UnprivilegedLogin), _managementConnection.Database); - _unprivilegedAppUser = new DatabaseUser(_managementConnection, _managementConnection.Database, _unprivilegedLogin); - _unprivilegedMasterUser = new DatabaseUser(_managementConnection, "master", _unprivilegedLogin); - - using (SqlCommand permissionsModificationCommand = _managementConnection.CreateCommand()) + // NOTE: If setup fails part way through, this constructor never returns, so xUnit never calls + // Dispose and the connection plus the login/users created so far would be leaked. A server + // login in particular is instance-wide, so it survives long after the test database is + // recreated. Opening the connection is inside the try for the same reason: a failed Open + // still leaves a SqlConnection instance that nothing else will ever dispose. + try { - permissionsModificationCommand.CommandText = $"DENY SELECT ON [master].[sys].[all_columns] TO {_unprivilegedMasterUser.Name}"; - permissionsModificationCommand.ExecuteNonQuery(); + _managementConnection.Open(); + + _unprivilegedLogin = new ServerLogin(_managementConnection, nameof(UnprivilegedLogin), _managementConnection.Database); + _unprivilegedAppUser = new DatabaseUser(_managementConnection, _managementConnection.Database, _unprivilegedLogin); + _unprivilegedMasterUser = new DatabaseUser(_managementConnection, "master", _unprivilegedLogin); + + using (SqlCommand permissionsModificationCommand = _managementConnection.CreateCommand()) + { + permissionsModificationCommand.CommandText = $"DENY SELECT ON [master].[sys].[all_columns] TO {_unprivilegedMasterUser.Name}"; + permissionsModificationCommand.ExecuteNonQuery(); - permissionsModificationCommand.CommandText = $"DENY SELECT ON [{_managementConnection.Database}].[sys].[all_columns] TO {_unprivilegedAppUser.Name}"; - permissionsModificationCommand.ExecuteNonQuery(); + permissionsModificationCommand.CommandText = $"DENY SELECT ON [{_managementConnection.Database}].[sys].[all_columns] TO {_unprivilegedAppUser.Name}"; + permissionsModificationCommand.ExecuteNonQuery(); + } + } + catch + { + Dispose(); + throw; } SqlConnectionStringBuilder tcpConnectionBuilder = new(DataTestUtility.TCPConnectionString) @@ -205,9 +219,24 @@ public void BulkCopyWithoutMetadataPermission_FailsWhenUsingAliases() public void Dispose() { - _unprivilegedAppUser?.Dispose(); - _unprivilegedMasterUser?.Dispose(); - _unprivilegedLogin?.Dispose(); + // Each drop is best-effort: a failure to remove one object must not prevent the others (in + // particular the instance-wide server login) from being removed. + DisposeSafely(_unprivilegedAppUser); + DisposeSafely(_unprivilegedMasterUser); + DisposeSafely(_unprivilegedLogin); + _managementConnection?.Dispose(); } + + private static void DisposeSafely(IDisposable? disposable) + { + try + { + disposable?.Dispose(); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(UnprivilegedLogin)}: cleanup failed: {ex.Message}"); + } + } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AADFedAuthTokenRefreshTest/AADFedAuthTokenRefreshTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AADFedAuthTokenRefreshTest/AADFedAuthTokenRefreshTest.cs index e5616776b1..c79b2edcb7 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AADFedAuthTokenRefreshTest/AADFedAuthTokenRefreshTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AADFedAuthTokenRefreshTest/AADFedAuthTokenRefreshTest.cs @@ -4,7 +4,13 @@ using System; using System.Diagnostics; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; using Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.Common.SystemDataInternals; +using Microsoft.Data.SqlClient.ManualTesting.Tests.SystemDataInternals; +using Microsoft.Data.SqlClient.Tests.Common; using Xunit; using Xunit.Abstractions; @@ -87,6 +93,107 @@ public void FedAuthTokenRefreshTest() } } + /// + /// Verifies both pools replace connections with expired or nearly expired tokens and + /// invoke the callback when the cached token also needs refreshing, for Open and OpenAsync. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsAADPasswordConnStrSetup))] + [InlineData(false, false, -1)] + [InlineData(false, true, -1)] + [InlineData(true, false, -1)] + [InlineData(true, true, -1)] + [InlineData(false, false, 5)] + [InlineData(false, true, 5)] + [InlineData(true, false, 5)] + [InlineData(true, true, 5)] + public async Task AccessTokenCallback_PooledConnectionIsReplacedOnExpiry(bool usePoolV2, bool async, int expiresInSeconds) + { + using var poolVersion = new ConnectionPoolVersionScope(usePoolV2); + string[] credentialKeys = { "Authentication", "User ID", "Password", "UID", "PWD" }; + var builder = new SqlConnectionStringBuilder( + DataTestUtility.RemoveKeysInConnStr(DataTestUtility.AADPasswordConnectionString, credentialKeys)) + { + Pooling = true, + MinPoolSize = 0, + MaxPoolSize = 1, + ConnectTimeout = 30, + Enlist = false + }; + var credential = DataTestUtility.GetTokenCredential(); + SqlAuthenticationToken callbackToken = null; + int callbackInvocations = 0; + using var connection = new SqlConnection(builder.ConnectionString) + { + AccessTokenCallback = async (parameters, cancellationToken) => + { + Interlocked.Increment(ref callbackInvocations); + const string suffix = "/.default"; + string scope = parameters.Resource.EndsWith(suffix) ? parameters.Resource : parameters.Resource + suffix; + AccessToken token = await credential.GetTokenAsync(new TokenRequestContext(new[] { scope }), cancellationToken); + callbackToken = new SqlAuthenticationToken(token.Token, token.ExpiresOn); + return callbackToken; + } + }; + + Task OpenConnection() + { + if (async) + { + return connection.OpenAsync(); + } + connection.Open(); + return Task.CompletedTask; + } + + // The empty pool requires a physical login, which invokes the callback and caches its token. + await OpenConnection(); + object original = connection.GetInternalConnection(); + Assert.NotNull(callbackToken); + int callbackCountAfterLogin = callbackInvocations; + Assert.True(callbackCountAfterLogin > 0); + // Close returns the physical connection to the pool; reopening reuses it without authentication. + connection.Close(); + await OpenConnection(); + Assert.Same(original, connection.GetInternalConnection()); + Assert.Equal(callbackCountAfterLogin, callbackInvocations); + + // The connection's expiry controls eviction; the cached token's expiry controls callback refresh. + // Age both without changing the real token or waiting. Five seconds is within the 30-second + // checkout buffer; minus one second covers an already expired token. + DateTimeOffset expiry = DateTimeOffset.UtcNow.AddSeconds(expiresInSeconds); + object cachedContext = FedAuthTokenHelper.GetAuthenticationContextValue(connection); + FieldInfo cacheExpiryField = cachedContext.GetType().GetField("_expirationTime", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(cacheExpiryField); + cacheExpiryField.SetValue(cachedContext, expiry.UtcDateTime); + FieldInfo tokenField = original.GetType().GetField("_fedAuthToken", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(tokenField); + object expiringToken = Activator.CreateInstance(tokenField.FieldType, + BindingFlags.Instance | BindingFlags.NonPublic, binder: null, + args: new object[] { new SqlAuthenticationToken(callbackToken.AccessToken, expiry) }, + culture: null); + tokenField.SetValue(original, expiringToken); + // Expiry is checked on checkout, not return, so Close does not invoke the callback. + connection.Close(); + Assert.Equal(callbackCountAfterLogin, callbackInvocations); + + // Checkout discards the expired physical connection rather than reauthenticating it. + // Its replacement needs a login, and the expiring cache entry forces another callback. + // The credential may still return the same valid token; callback invocation is what matters. + await OpenConnection(); + Assert.NotSame(original, connection.GetInternalConnection()); + Assert.True(callbackInvocations > callbackCountAfterLogin); + object replacement = connection.GetInternalConnection(); + int callbackCountAfterRefresh = callbackInvocations; + // The replacement now has a valid token, so another reopen reuses it without another callback. + connection.Close(); + await OpenConnection(); + Assert.Same(replacement, connection.GetInternalConnection()); + Assert.Equal(callbackCountAfterRefresh, callbackInvocations); + using SqlCommand command = connection.CreateCommand(); + command.CommandText = "SELECT 1"; + Assert.Equal(1, async ? await command.ExecuteScalarAsync() : command.ExecuteScalar()); + } + [Conditional("DEBUG")] private void LogInfo(string message) { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AdapterTest/AdapterTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AdapterTest/AdapterTest.cs index 960f3ee822..d9c86e8eff 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AdapterTest/AdapterTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AdapterTest/AdapterTest.cs @@ -109,7 +109,7 @@ public void SimpleFillTest() // TODO Synapse: Remove Northwind dependency by creating required tables in setup. [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] // https://github.com/dotnet/SqlClient/issues/4135 - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public void FillShouldAllowRetryLogicProviderToBeInvoked() { int maxRetries = 3; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AsyncTest/AsyncTimeoutTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AsyncTest/AsyncTimeoutTest.cs index 1c0abdc5d7..69753aed41 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AsyncTest/AsyncTimeoutTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/AsyncTest/AsyncTimeoutTest.cs @@ -29,7 +29,7 @@ public enum AsyncAPI // Synapse: WAITFOR DELAY not supported [Parse error at line: 1, column: 1: Incorrect syntax near 'WAITFOR'.] [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] [ClassData(typeof(AsyncTimeoutTestVariations))] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public static void TestDelayedAsyncTimeout(AsyncAPI api, string commonObj, int delayPeriod, bool marsEnabled) => RunTest(api, commonObj, delayPeriod, marsEnabled); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionPoolTest/ConnectionPoolTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionPoolTest/ConnectionPoolTest.cs index a3ae028a5d..1e46f0f744 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionPoolTest/ConnectionPoolTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionPoolTest/ConnectionPoolTest.cs @@ -65,7 +65,7 @@ public static class ConnectionPoolTest /// [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] [ClassData(typeof(ConnectionPoolConnectionStringProvider))] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public static void BasicConnectionPoolingTest(string connectionString) { SqlConnection.ClearAllPools(); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionPoolTest/TransactionPoolTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionPoolTest/TransactionPoolTest.cs index 381d35bf02..e474493b30 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionPoolTest/TransactionPoolTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionPoolTest/TransactionPoolTest.cs @@ -86,7 +86,7 @@ public static void BasicTransactionPoolTest(string connectionString) // Expected: 2 // Actual: 3 // at Microsoft.Data.SqlClient.ManualTesting.Tests.TransactionPoolTest.TransactionCleanupTest(String connectionString) in TransactionPoolTest.cs:line 101 - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] [ClassData(typeof(ConnectionPoolConnectionStringProvider))] public static void TransactionCleanupTest(string connectionString) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionTestWithSSLCert/CertificateTestWithTdsServer.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionTestWithSSLCert/CertificateTestWithTdsServer.cs index 828a87d63e..0dbb7cca8a 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionTestWithSSLCert/CertificateTestWithTdsServer.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectionTestWithSSLCert/CertificateTestWithTdsServer.cs @@ -64,7 +64,18 @@ public CertificateTestWithTdsServer() Directory.CreateDirectory(s_fullPathToClientCert); } - RunPowershellScript(s_fullPathToPowershellScript); + // NOTE: The setup script installs a certificate into the LocalMachine root store and writes + // the scratch files. If it fails part way through, xUnit never calls Dispose, so run the + // cleanup here to avoid leaving the certificate behind. + try + { + RunPowershellScript(s_fullPathToPowershellScript); + } + catch + { + Dispose(); + throw; + } } private static bool IsLocalHost() @@ -254,18 +265,43 @@ private X509Certificate2 GetEncryptionCertificate(string fileName, string? passw private void RemoveCertificate() { - string thumbprint = File.ReadAllText(s_fullPathTothumbprint); - using X509Store certStore = new(StoreName.Root, StoreLocation.LocalMachine); - certStore.Open(OpenFlags.ReadWrite); - X509Certificate2Collection certCollection = certStore.Certificates.Find(X509FindType.FindByThumbprint, thumbprint, false); - if (certCollection.Count > 0) + // NOTE: Each step is independent. Previously a missing thumbprint file (or a failed store + // removal) aborted the whole cleanup, leaving the certificate in the LocalMachine root + // store and the scratch directory on disk. + try + { + if (File.Exists(s_fullPathTothumbprint)) + { + string thumbprint = File.ReadAllText(s_fullPathTothumbprint); + using X509Store certStore = new(StoreName.Root, StoreLocation.LocalMachine); + certStore.Open(OpenFlags.ReadWrite); + X509Certificate2Collection certCollection = certStore.Certificates.Find(X509FindType.FindByThumbprint, thumbprint, false); + if (certCollection.Count > 0) + { + certStore.Remove(certCollection[0]); + } + certStore.Close(); + } + } + catch (Exception ex) { - certStore.Remove(certCollection[0]); + Console.WriteLine($"{nameof(CertificateTestWithTdsServer)}: failed to remove certificate: {ex.Message}"); } - certStore.Close(); - File.Delete(s_fullPathTothumbprint); - Directory.Delete(s_fullPathToClientCert, true); + TryCleanup(() => File.Delete(s_fullPathTothumbprint)); + TryCleanup(() => Directory.Delete(s_fullPathToClientCert, true)); + } + + private static void TryCleanup(Action cleanupAction) + { + try + { + cleanupAction(); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(CertificateTestWithTdsServer)}: cleanup step failed: {ex.Message}"); + } } private static void RemoveForceEncryptionFromRegistryPath(string registryPath) @@ -292,8 +328,17 @@ protected virtual void Dispose(bool disposing) { if (disposing && !string.IsNullOrEmpty(s_fullPathTothumbprint)) { - RemoveCertificate(); - RemoveForceEncryptionFromRegistryPath(ForceEncryptionRegistryPath); + // NOTE: The registry/service reset must run even if certificate removal fails, + // otherwise the SQL Server instance is left with ForceEncryption enabled and a + // reference to a certificate that may no longer exist. + try + { + RemoveCertificate(); + } + finally + { + RemoveForceEncryptionFromRegistryPath(ForceEncryptionRegistryPath); + } } } else diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectivityTests/ConnectivityTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectivityTests/ConnectivityTest.cs index fbbee8db26..3b42d7b1c8 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectivityTests/ConnectivityTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ConnectivityTests/ConnectivityTest.cs @@ -24,10 +24,10 @@ public static class ConnectivityTest private static readonly string s_dbConnectionString = new SqlConnectionStringBuilder(s_connectionString) { InitialCatalog = s_databaseName }.ConnectionString; private static readonly string s_createDatabaseCmd = $"CREATE DATABASE {s_databaseName}"; private static readonly string s_createTableCmd = $"CREATE TABLE {s_tableName} (NAME NVARCHAR(40), AGE INT)"; - private static readonly string s_alterDatabaseSingleCmd = $"ALTER DATABASE {s_databaseName} SET SINGLE_USER WITH ROLLBACK IMMEDIATE;"; - private static readonly string s_alterDatabaseMultiCmd = $"ALTER DATABASE {s_databaseName} SET MULTI_USER WITH ROLLBACK IMMEDIATE;"; + private static readonly string s_alterDatabaseSingleCmd = $"IF (EXISTS(SELECT 1 FROM sys.databases WHERE name = '{s_databaseName}')) ALTER DATABASE {s_databaseName} SET SINGLE_USER WITH ROLLBACK IMMEDIATE;"; + private static readonly string s_alterDatabaseMultiCmd = $"IF (EXISTS(SELECT 1 FROM sys.databases WHERE name = '{s_databaseName}')) ALTER DATABASE {s_databaseName} SET MULTI_USER WITH ROLLBACK IMMEDIATE;"; private static readonly string s_selectTableCmd = $"SELECT COUNT(*) FROM {s_tableName}"; - private static readonly string s_dropDatabaseCmd = $"DROP DATABASE {s_databaseName}"; + private static readonly string s_dropDatabaseCmd = $"IF (EXISTS(SELECT 1 FROM sys.databases WHERE name = '{s_databaseName}')) DROP DATABASE {s_databaseName}"; // Synapse: Stored procedure sp_who2 does not exist or is not supported. // Synapse: SqlConnection.ServerProcessId is always retrieved as 0. @@ -263,8 +263,19 @@ public static void ConnectionKilledTest() } finally { - // Kill all the connections, set Database to SINGLE_USER Mode and drop Database - DataTestUtility.RunNonQuery(s_connectionString, s_alterDatabaseSingleCmd, 4); + // Kill all the connections, set Database to SINGLE_USER Mode and drop Database. + // NOTE: The database name embeds a GUID, so failing to drop it leaks a database that + // nothing will ever reclaim. Switching to SINGLE_USER is only an optimization to + // evict other sessions, so its failure must not prevent the drop from running. + try + { + DataTestUtility.RunNonQuery(s_connectionString, s_alterDatabaseSingleCmd, 4); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(ConnectionKilledTest)}: failed to set '{s_databaseName}' to SINGLE_USER: {ex.Message}"); + } + DataTestUtility.RunNonQuery(s_connectionString, s_dropDatabaseCmd, 4); } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/DataReaderTest/DataReaderTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/DataReaderTest/DataReaderTest.cs index 2a2132dbcb..65d17b8497 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/DataReaderTest/DataReaderTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/DataReaderTest/DataReaderTest.cs @@ -966,7 +966,7 @@ public _Row(int id, Guid documentIdentificationId, string name, string value) // reading 21 rows that include several nvarchar(max) values tens-of-thousands of characters // long. With small packet sizes the large strings require many network round-trips, and on // non-Windows platforms the managed SNI layer is used which adds further overhead. - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] public static async Task CanReadAwkwardDataLengths() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/DataStreamTest/DataStreamTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/DataStreamTest/DataStreamTest.cs index a3330a371b..6ee68d192e 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/DataStreamTest/DataStreamTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/DataStreamTest/DataStreamTest.cs @@ -201,7 +201,7 @@ public static void GetXmlReader_ReturnsXmlReaderCorrectly(string connectionStrin // Enumeration is disabled to prevent generating empty test set when connection strings are not setup. [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] [MemberData(nameof(ConnectionStrings), DisableDiscoveryEnumeration = true)] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public static void ReadStream_ReadsStreamDataCorrectly(string connectionString) { ReadStream(connectionString); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/DirectionalityTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/DirectionalityTest.cs new file mode 100644 index 0000000000..f18cc94d06 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/DirectionalityTest.cs @@ -0,0 +1,251 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests; + +/// +/// Verifies that SqlClient preserves the logical UTF-16 representation of bidirectional text. +/// Visual direction, shaping, and mirroring are responsibilities of the consuming UI. +/// +[Trait("Set", "3")] +public sealed class DirectionalityTest +{ + private static readonly string[] s_bidiText = + { + "\u0645\u0631\u062D\u0628\u0627 Microsoft 01 - \u0639\u0627\u0644\u0645 \U0001F310", + "\u05E9\u05DC\u05D5\u05DD Microsoft 01 - \u05E2\u05D5\u05DC\u05DD \U0001F310" + }; + + /// + /// Ensures mixed Arabic/Hebrew, Latin, numeric, punctuation, and supplementary characters + /// round-trip unchanged through parameters, readers, sequential streaming, and bulk copy. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] + [InlineData(false)] + [InlineData(true)] + public async Task BidiText_RoundTripsWithoutTransformation(bool async) + { + using SqlConnection setupConnection = new(DataTestUtility.TCPConnectionString); + await OpenConnection(setupConnection, async); + + using Table sourceTable = new( + setupConnection, + "DirectionalitySource", + "(Id int NOT NULL, Value nvarchar(max) NOT NULL)"); + using Table destinationTable = new( + setupConnection, + "DirectionalityDestination", + "(Id int NOT NULL, Value nvarchar(max) NOT NULL)"); + + await InsertValues(setupConnection, sourceTable.Name, async); + await VerifyOrdinaryReader(setupConnection, sourceTable.Name, async); + await VerifyGetChars(setupConnection, sourceTable.Name, async); + await VerifyTextReader(setupConnection, sourceTable.Name, async); + await CopyValues(sourceTable.Name, destinationTable.Name, async); + await VerifyOrdinaryReader(setupConnection, destinationTable.Name, async); + } + + /// + /// Opens a connection through the requested synchronous or asynchronous API. + /// + /// The connection to open. + /// Whether to use the asynchronous API. + /// A task representing the open operation. + private static async Task OpenConnection(SqlConnection connection, bool async) + { + if (async) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + } + + /// + /// Inserts the bidi samples through explicitly typed Unicode parameters. + /// + /// The open connection used to insert the samples. + /// The table that receives the samples. + /// Whether to use asynchronous command execution. + /// A task representing the insert operations. + private static async Task InsertValues(SqlConnection connection, string tableName, bool async) + { + using SqlCommand command = new($"INSERT INTO {tableName} (Id, Value) VALUES (@id, @value)", connection); + SqlParameter idParameter = command.Parameters.Add("@id", SqlDbType.Int); + SqlParameter valueParameter = command.Parameters.Add("@value", SqlDbType.NVarChar, -1); + + for (int index = 0; index < s_bidiText.Length; index++) + { + idParameter.Value = index; + valueParameter.Value = s_bidiText[index]; + + if (async) + { + await command.ExecuteNonQueryAsync(); + } + else + { + command.ExecuteNonQuery(); + } + } + } + + /// + /// Reads complete strings through ordinary reader accessors and compares their UTF-16 content. + /// + /// The open connection used to read the samples. + /// The table containing the samples. + /// Whether to use asynchronous reader APIs. + /// A task representing the verification operation. + private static async Task VerifyOrdinaryReader(SqlConnection connection, string tableName, bool async) + { + using SqlCommand command = new($"SELECT Id, Value FROM {tableName} ORDER BY Id", connection); + using SqlDataReader reader = async + ? await command.ExecuteReaderAsync() + : command.ExecuteReader(); + + for (int index = 0; index < s_bidiText.Length; index++) + { + bool hasRow = async ? await reader.ReadAsync() : reader.Read(); + Assert.True(hasRow); + Assert.Equal(index, reader.GetInt32(0)); + AssertOrdinalEqual(s_bidiText[index], reader.GetString(1)); + string fieldValue = async + ? await reader.GetFieldValueAsync(1) + : reader.GetFieldValue(1); + AssertOrdinalEqual(s_bidiText[index], fieldValue); + } + + Assert.False(async ? await reader.ReadAsync() : reader.Read()); + } + + /// + /// Reads one UTF-16 code unit at a time to cover direction and surrogate boundaries in GetChars. + /// + /// The open connection used to read the samples. + /// The table containing the samples. + /// Whether to use asynchronous reader execution. + /// A task representing the verification operation. + private static async Task VerifyGetChars(SqlConnection connection, string tableName, bool async) + { + using SqlCommand command = new($"SELECT Value FROM {tableName} ORDER BY Id", connection); + using SqlDataReader reader = async + ? await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : command.ExecuteReader(CommandBehavior.SequentialAccess); + + for (int index = 0; index < s_bidiText.Length; index++) + { + bool hasRow = async ? await reader.ReadAsync() : reader.Read(); + Assert.True(hasRow); + + StringBuilder result = new(); + char[] buffer = new char[1]; + long dataIndex = 0; + long charsRead; + do + { + charsRead = reader.GetChars(0, dataIndex, buffer, 0, buffer.Length); + result.Append(buffer, 0, (int)charsRead); + dataIndex += charsRead; + } + while (charsRead != 0); + + AssertOrdinalEqual(s_bidiText[index], result.ToString()); + } + } + + /// + /// Reads bidi text through the sequential TextReader using small sync or async buffer operations. + /// + /// The open connection used to read the samples. + /// The table containing the samples. + /// Whether to use asynchronous reader and text operations. + /// A task representing the verification operation. + private static async Task VerifyTextReader(SqlConnection connection, string tableName, bool async) + { + using SqlCommand command = new($"SELECT Value FROM {tableName} ORDER BY Id", connection); + using SqlDataReader reader = async + ? await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : command.ExecuteReader(CommandBehavior.SequentialAccess); + + for (int index = 0; index < s_bidiText.Length; index++) + { + bool hasRow = async ? await reader.ReadAsync() : reader.Read(); + Assert.True(hasRow); + + using TextReader textReader = reader.GetTextReader(0); + StringBuilder result = new(); + char[] buffer = new char[2]; + int charsRead; + do + { + charsRead = async + ? await textReader.ReadAsync(buffer, 0, buffer.Length) + : textReader.Read(buffer, 0, buffer.Length); + result.Append(buffer, 0, charsRead); + } + while (charsRead != 0); + + AssertOrdinalEqual(s_bidiText[index], result.ToString()); + } + } + + /// + /// Copies the Unicode rows through streaming SqlBulkCopy using the requested execution mode. + /// + /// The table containing the source rows. + /// The table receiving the copied rows. + /// Whether to use asynchronous reader and bulk-copy APIs. + /// A task representing the copy operation. + private static async Task CopyValues(string sourceTableName, string destinationTableName, bool async) + { + using SqlConnection sourceConnection = new(DataTestUtility.TCPConnectionString); + using SqlConnection destinationConnection = new(DataTestUtility.TCPConnectionString); + await OpenConnection(sourceConnection, async); + await OpenConnection(destinationConnection, async); + + using SqlCommand command = new($"SELECT Id, Value FROM {sourceTableName} ORDER BY Id", sourceConnection); + using SqlDataReader reader = async + ? await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : command.ExecuteReader(CommandBehavior.SequentialAccess); + using SqlBulkCopy bulkCopy = new(destinationConnection) + { + DestinationTableName = destinationTableName, + EnableStreaming = true + }; + bulkCopy.ColumnMappings.Add(0, 0); + bulkCopy.ColumnMappings.Add(1, 1); + + if (async) + { + await bulkCopy.WriteToServerAsync(reader); + } + else + { + bulkCopy.WriteToServer(reader); + } + } + + /// + /// Compares strings ordinally so the assertion checks logical storage rather than visual rendering. + /// + /// The original UTF-16 string. + /// The round-tripped UTF-16 string. + private static void AssertOrdinalEqual(string expected, string actual) + { + Assert.True( + string.Equals(expected, actual, StringComparison.Ordinal), + $"Expected and actual UTF-16 values differ. Expected length: {expected.Length}; actual length: {actual?.Length}."); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/GlobalizationEncodingTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/GlobalizationEncodingTests.cs new file mode 100644 index 0000000000..f2ba885b1f --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/GlobalizationTest/GlobalizationEncodingTests.cs @@ -0,0 +1,217 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects; +using Xunit; + +namespace Microsoft.Data.SqlClient.ManualTesting.Tests +{ + /// + /// Provides representative worldwide text for encoding validation tests. + /// + internal static class GlobalizationTestData + { + internal const string RepresentativeText = + "Latin: Caf\u00E9 / Cafe\u0301 | " + + "Arabic: \u0627\u0644\u0639\u064E\u0631\u064E\u0628\u0650\u064A\u064E\u0651\u0629 | " + + "Devanagari: \u0939\u093F\u0928\u094D\u0926\u0940 | " + + "Thai: \u0E20\u0E32\u0E29\u0E32\u0E44\u0E17\u0E22 | " + + "CJK: \u65E5\u672C\u8A9E \u4E2D\u6587 \uD55C\uAD6D\uC5B4 | " + + "Supplementary: \uD83D\uDE00 \uD834\uDD1E | " + + "Emoji sequence: \uD83D\uDC69\uD83C\uDFFD\u200D\uD83D\uDCBB"; + + /// + /// Repeats the representative text so reads span TDS packets and internal character buffers. + /// + internal static string CreatePacketSpanningText() + { + StringBuilder value = new(); + for (int i = 0; i < 16; i++) + { + value.Append(RepresentativeText); + } + + return value.ToString(); + } + } + + /// + /// Validates exact worldwide text preservation through parameters, SQL Server storage, and reader APIs. + /// + [Trait("Set", "3")] + public static class GlobalizationEncodingTests + { + /// + /// Verifies normal and streamed Unicode parameters round-trip exactly through buffered and sequential + /// readers on synchronous and asynchronous paths. + /// + /// Whether command and reader operations use asynchronous APIs. + /// Whether the input parameter is supplied through a . + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public static async Task UnicodeParameterAndReaderRoundTrip_PreservesWorldwideText(bool useAsync, bool streamInput) + { + string expected = GlobalizationTestData.CreatePacketSpanningText(); + SqlConnectionStringBuilder connectionString = new(DataTestUtility.TCPConnectionString) + { + PacketSize = 512 + }; + + using SqlConnection connection = new(connectionString.ConnectionString); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + + using Table table = new(connection, nameof(UnicodeParameterAndReaderRoundTrip_PreservesWorldwideText), "(Value nvarchar(max) NOT NULL)"); + using StringReader inputReader = new(expected); + using (SqlCommand insert = new($"INSERT INTO {table.Name} (Value) VALUES (@value)", connection)) + { + insert.Parameters.Add(new SqlParameter("@value", SqlDbType.NVarChar, -1) + { + Value = streamInput ? inputReader : expected + }); + + if (useAsync) + { + await insert.ExecuteNonQueryAsync(); + } + else + { + insert.ExecuteNonQuery(); + } + } + + string query = $"SELECT Value FROM {table.Name}"; + string directValue = await ReadDirectValue(connection, query, useAsync); + string streamedValue = await ReadStreamedValue(connection, query, useAsync); + + Assert.Equal(expected, directValue); + Assert.Equal(expected, streamedValue); + Assert.Equal(expected.Length, directValue.Length); + Assert.Equal(expected.Length, streamedValue.Length); + } + + /// + /// Verifies a UTF-8-collated varchar value returns exact worldwide text and the expected UTF-8 bytes + /// on synchronous and asynchronous paths. + /// + /// Whether command and reader operations use asynchronous APIs. + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsUTF8Supported))] + [InlineData(false)] + [InlineData(true)] + public static async Task Utf8VarcharRoundTrip_PreservesWorldwideTextAndBytes(bool useAsync) + { + string expected = GlobalizationTestData.CreatePacketSpanningText(); + SqlConnectionStringBuilder connectionString = new(DataTestUtility.TCPConnectionString) + { + PacketSize = 512 + }; + using SqlConnection connection = new(connectionString.ConnectionString); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + + using Table table = new( + connection, + nameof(Utf8VarcharRoundTrip_PreservesWorldwideTextAndBytes), + "(Value varchar(max) COLLATE Latin1_General_100_CS_AS_KS_WS_SC_UTF8 NOT NULL)"); + using (SqlCommand insert = new($"INSERT INTO {table.Name} (Value) VALUES (@value)", connection)) + { + insert.Parameters.Add(new SqlParameter("@value", SqlDbType.NVarChar, -1) { Value = expected }); + if (useAsync) + { + await insert.ExecuteNonQueryAsync(); + } + else + { + insert.ExecuteNonQuery(); + } + } + + using SqlCommand select = new($"SELECT Value, CONVERT(varbinary(max), Value) FROM {table.Name}", connection); + using SqlDataReader reader = useAsync + ? await select.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : select.ExecuteReader(CommandBehavior.SequentialAccess); + bool hasRow = useAsync ? await reader.ReadAsync() : reader.Read(); + + Assert.True(hasRow); + Assert.Equal(expected, reader.GetString(0)); + Assert.Equal(Encoding.UTF8.GetBytes(expected), reader.GetFieldValue(1)); + Assert.False(useAsync ? await reader.ReadAsync() : reader.Read()); + } + + /// + /// Reads a string through the standard buffered reader path. + /// + /// The open SQL connection used to execute the query. + /// The query that returns one string value. + /// Whether command and reader operations use asynchronous APIs. + /// The string returned by SQL Server. + private static async Task ReadDirectValue(SqlConnection connection, string query, bool useAsync) + { + using SqlCommand command = new(query, connection); + using SqlDataReader reader = useAsync + ? await command.ExecuteReaderAsync() + : command.ExecuteReader(); + bool hasRow = useAsync ? await reader.ReadAsync() : reader.Read(); + + Assert.True(hasRow); + string result = reader.GetString(0); + Assert.False(useAsync ? await reader.ReadAsync() : reader.Read()); + return result; + } + + /// + /// Reads a string through sequential calls with a small buffer so character + /// sequences cross read boundaries. + /// + /// The open SQL connection used to execute the query. + /// The query that returns one string value. + /// Whether command and reader operations use asynchronous APIs. + /// The string returned by SQL Server. + private static async Task ReadStreamedValue(SqlConnection connection, string query, bool useAsync) + { + using SqlCommand command = new(query, connection); + using SqlDataReader reader = useAsync + ? await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess) + : command.ExecuteReader(CommandBehavior.SequentialAccess); + bool hasRow = useAsync ? await reader.ReadAsync() : reader.Read(); + + Assert.True(hasRow); + using TextReader textReader = reader.GetTextReader(0); + char[] buffer = new char[3]; + StringBuilder result = new(); + int charsRead; + do + { + charsRead = useAsync + ? await textReader.ReadAsync(buffer, 0, buffer.Length) + : textReader.Read(buffer, 0, buffer.Length); + result.Append(buffer, 0, charsRead); + } + while (charsRead != 0); + + Assert.False(useAsync ? await reader.ReadAsync() : reader.Read()); + return result.ToString(); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonBulkCopyTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonBulkCopyTest.cs index 4e5fa29c7d..629ce3785e 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonBulkCopyTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonBulkCopyTest.cs @@ -15,7 +15,7 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.JsonTest { [Trait("Set", "3")] - public class JsonBulkCopyTest + public class JsonBulkCopyTest : IDisposable { private readonly ITestOutputHelper _output; private static readonly string _generatedJsonFile = DataTestUtility.GetShortName("randomRecords"); @@ -28,6 +28,37 @@ public JsonBulkCopyTest(ITestOutputHelper output) _output = output; } + /// + /// Drops the tables and removes the scratch files created by the tests. + /// + /// + /// The table names embed a GUID, so without this the tests left two tables behind in the + /// shared test database on every run. + /// + public void Dispose() + { + try + { + if (DataTestUtility.AreConnStringsSetup() && DataTestUtility.IsJsonSupported) + { + using SqlConnection connection = new SqlConnection(DataTestUtility.TCPConnectionString); + connection.Open(); + + DataTestUtility.DropTable(connection, _sourceTableName); + DataTestUtility.DropTable(connection, _destinationTableName); + } + } + catch (Exception ex) + { + _output.WriteLine($"{nameof(JsonBulkCopyTest)}: failed to drop test tables: {ex.Message}"); + } + finally + { + DeleteFile(_generatedJsonFile); + DeleteFile(_outputFile); + } + } + public static IEnumerable JsonBulkCopyTestData() { yield return new object[] { CommandBehavior.Default, false, 30, 10 }; diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/MARSSessionPoolingTest/MarsSessionPoolingTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/MARSSessionPoolingTest/MarsSessionPoolingTest.cs index 2fca930559..18b08c8b07 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/MARSSessionPoolingTest/MarsSessionPoolingTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/MARSSessionPoolingTest/MarsSessionPoolingTest.cs @@ -192,7 +192,7 @@ public void ExecuteReader_DisposeReader(CommandType commandType) } } - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsNotManagedInstance))] [InlineData(CommandType.Text)] [InlineData(CommandType.StoredProcedure)] @@ -228,7 +228,7 @@ public void ExecuteReader_GarbageCollectReader(CommandType commandType) } } - [Trait("Category", "flaky")] // Assert.Equal() Failure: Values differ + [Trait("category", "flaky")] // Assert.Equal() Failure: Values differ // Expected: 5 // Actual: 4 [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsNotManagedInstance))] @@ -301,7 +301,7 @@ public void ExecuteReader_CloseConnection(CommandType commandType) } } - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse), nameof(DataTestUtility.IsNotManagedInstance))] [InlineData(CommandType.Text)] [InlineData(CommandType.StoredProcedure)] diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParallelTransactionsTest/ParallelTransactionsTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParallelTransactionsTest/ParallelTransactionsTest.cs index 6f3f74fe4e..4f59823b79 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParallelTransactionsTest/ParallelTransactionsTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParallelTransactionsTest/ParallelTransactionsTest.cs @@ -43,25 +43,21 @@ private static void BasicParallelTest(string connectionString, string tempTableN SqlTransaction trans2 = connection.BeginTransaction(); SqlTransaction trans3 = connection.BeginTransaction(); - SqlCommand com1 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); + using SqlCommand com1 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); com1.Transaction = trans1; com1.ExecuteNonQuery(); - SqlCommand com2 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); + using SqlCommand com2 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); com2.Transaction = trans2; com2.ExecuteNonQuery(); - SqlCommand com3 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); + using SqlCommand com3 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); com3.Transaction = trans3; com3.ExecuteNonQuery(); trans1.Rollback(); trans2.Rollback(); trans3.Rollback(); - - com1.Dispose(); - com2.Dispose(); - com3.Dispose(); } } @@ -100,15 +96,15 @@ private static void MultipleExecutesInSameTransactionTest(string connectionStrin SqlTransaction trans2 = connection.BeginTransaction(); SqlTransaction trans3 = connection.BeginTransaction(); - SqlCommand com1 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); + using SqlCommand com1 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); com1.Transaction = trans1; com1.ExecuteNonQuery(); - SqlCommand com2 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); + using SqlCommand com2 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); com2.Transaction = trans2; com2.ExecuteNonQuery(); - SqlCommand com3 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); + using SqlCommand com3 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); com3.Transaction = trans3; com3.ExecuteNonQuery(); @@ -116,15 +112,14 @@ private static void MultipleExecutesInSameTransactionTest(string connectionStrin trans2.Rollback(); trans3.Rollback(); - com1.Dispose(); - com2.Dispose(); - com3.Dispose(); - - SqlCommand com4 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); + using SqlCommand com4 = new SqlCommand("select top 1 EmployeeID from " + tempTableName, connection); com4.Transaction = trans1; - SqlDataReader reader4 = com4.ExecuteReader(); - reader4.Dispose(); - com4.Dispose(); + using (SqlDataReader reader4 = com4.ExecuteReader()) + { + // Scoped deliberately: MARS is off here, so the reader must be closed before + // the rollback below, which would otherwise fail on a connection that still + // has an open reader. + } trans1.Rollback(); } @@ -138,7 +133,7 @@ private static string CreateTempTable(string connectionString) using (var conn = new SqlConnection(connectionString)) { conn.Open(); - SqlCommand cmd = new SqlCommand(string.Format("SELECT EmployeeID, LastName, FirstName, Title, Address, City, Region, PostalCode, Country into {0} from Employees", tempTableName), conn); + using SqlCommand cmd = new SqlCommand(string.Format("SELECT EmployeeID, LastName, FirstName, Title, Address, City, Region, PostalCode, Country into {0} from Employees", tempTableName), conn); cmd.ExecuteNonQuery(); cmd.CommandText = string.Format("alter table {0} add constraint EmployeeID_{1} primary key (EmployeeID)", tempTableName, uniqueKey); cmd.ExecuteNonQuery(); @@ -147,13 +142,26 @@ private static string CreateTempTable(string connectionString) return tempTableName; } + /// + /// Best-effort: both callers invoke this from a finally, so a throw here would surface + /// in place of the failure that is already propagating and hide the real test result. The + /// table is named on failure so the leak stays attributable. + /// private static void DropTempTable(string connectionString, string tempTableName) { - using (SqlConnection con1 = new SqlConnection(connectionString)) + try { - con1.Open(); - SqlCommand cmd = new SqlCommand("Drop table " + tempTableName, con1); - cmd.ExecuteNonQuery(); + using (SqlConnection con1 = new SqlConnection(connectionString)) + { + con1.Open(); + using SqlCommand cmd = new SqlCommand( + string.Format("IF (OBJECT_ID('{0}') IS NOT NULL) DROP TABLE {0}", tempTableName), con1); + cmd.ExecuteNonQuery(); + } + } + catch (Exception ex) + { + Console.WriteLine($"Failed to drop temp table '{tempTableName}'; it may be orphaned in the test database. {ex}"); } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/DateTimeVariantTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/DateTimeVariantTests.cs index e968295b5a..510a2d0608 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/DateTimeVariantTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/DateTimeVariantTests.cs @@ -52,7 +52,7 @@ public TestResult(object value, string baseTypeName) /// Tests for DateTime variant parameters with different date/time types. /// [Trait("Set", "3")] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public class DateTimeVariantTests { private static void RunTest( diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/ParametersTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/ParametersTest.cs index 01bbf586fc..77b58ce09b 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/ParametersTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/ParametersTest.cs @@ -244,7 +244,7 @@ public static void Test_WithGuidValue_ShouldReturnGuid() } // Synapse: Parse error at line: 1, column: 8: Incorrect syntax near 'TYPE'. - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] public static void TestParametersWithDatatablesTVPInsert() { @@ -287,7 +287,7 @@ public static void TestParametersWithDatatablesTVPInsert() #if !NETFRAMEWORK // Synapse: Parse error at line: 1, column: 8: Incorrect syntax near 'TYPE'. - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] public static void TestParametersWithSqlRecordsTVPInsert() { @@ -344,7 +344,7 @@ public static void TestParametersWithSqlRecordsTVPInsert() Assert.Equal(2, count); } - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] public static void TestDateOnlyTVPDataTable_CommandSP() { @@ -379,7 +379,7 @@ public static void TestDateOnlyTVPDataTable_CommandSP() cmd.ExecuteNonQuery(); } - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] public static void TestDateOnlyTVPSqlDataRecord_CommandSP() { @@ -532,11 +532,112 @@ public static void TestScaledDecimalParameter_CommandInsert(string connectionStr Assert.True(ValidateInsertedValues(connection, decimalTable.Name, truncateScaledDecimal), $"Invalid test happened with connection string [{connection.ConnectionString}]"); } - [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] - public static void TestOutOfRangeDecimalParameter_CommandSelect() + /// + /// CLR and SQL zero values with different representations must round-trip into decimal(p,p) columns. + /// One async case covers command execution parity; scale boundaries use the shared synchronous conversion. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTCPConnStringSetup))] + [InlineData(1, false)] + [InlineData(2, false)] + [InlineData(3, false)] + [InlineData(3, true)] + [InlineData(28, false)] + [InlineData(29, false)] + [InlineData(38, false)] + public static async Task ZeroDecimalParameter_CommandInsert(byte scale, bool useAsync) { using SqlConnection connection = new(DataTestUtility.TCPConnectionString); - connection.Open(); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + + using Table table = new(connection, "ZeroDecimalParameter", $"([Value] decimal({scale},{scale}))"); + using SqlCommand command = new( + $"INSERT INTO {table.Name} ([Value]) OUTPUT INSERTED.[Value] VALUES (@Value)", connection); + SqlParameter parameter = command.Parameters.Add("@Value", SqlDbType.Decimal); + parameter.Precision = scale; + parameter.Scale = scale; + + foreach (object value in new object[] + { + 0m, 0.0m, 0.000m, new decimal(0, 0, 0, true, 0), + new SqlDecimal(0m), new SqlDecimal(0.000m), + new SqlDecimal(new decimal(0, 0, 0, true, 0)), + new SqlDecimal(38, scale, true, 0, 0, 0, 0), + new SqlDecimal(38, scale, false, 0, 0, 0, 0) + }) + { + parameter.Value = value; + using SqlDataReader reader = useAsync + ? await command.ExecuteReaderAsync() + : command.ExecuteReader(); + Assert.True(useAsync ? await reader.ReadAsync() : reader.Read()); + SqlDecimal actual = reader.GetSqlDecimal(0); + Assert.Equal(scale, actual.Precision); + Assert.Equal(scale, actual.Scale); + Assert.Equal(0, actual.CompareTo(new SqlDecimal(0))); + } + } + + /// + /// Correcting zero precision must not allow nonzero values that exceed the parameter precision. + /// Both command APIs must surface the same exception, including when the async result is awaited. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTCPConnStringSetup))] + [InlineData(false)] + [InlineData(true)] + public static async Task DecimalParameter_RejectsInsufficientPrecision(bool useAsync) + { + using SqlConnection connection = new(DataTestUtility.TCPConnectionString); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } + + using SqlCommand command = new("SELECT @Value", connection); + SqlParameter parameter = command.Parameters.Add("@Value", SqlDbType.Decimal); + parameter.Precision = 3; + parameter.Scale = 3; + foreach (object value in new object[] { 1m, -1m, new SqlDecimal(1m), new SqlDecimal(-1m) }) + { + parameter.Value = value; + if (useAsync) + { + await Assert.ThrowsAsync(() => command.ExecuteNonQueryAsync()); + } + else + { + Assert.Throws(() => command.ExecuteNonQuery()); + } + } + } + + /// + /// Large decimal values must still round-trip after rescaling beyond the CLR decimal capacity. + /// + [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.IsTCPConnStringSetup))] + [InlineData(false)] + [InlineData(true)] + public static async Task TestOutOfRangeDecimalParameter_CommandSelect(bool useAsync) + { + using SqlConnection connection = new(DataTestUtility.TCPConnectionString); + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } using SqlCommand cmd = new("SELECT @Value", connection); // A System.Decimal value has a maximum precision of 29 digits. We specify a Precision of 38 and a Scale of 2 in order @@ -550,9 +651,11 @@ public static void TestOutOfRangeDecimalParameter_CommandSelect() cmd.Parameters.Add(p); - using SqlDataReader reader = cmd.ExecuteReader(); + using SqlDataReader reader = useAsync + ? await cmd.ExecuteReaderAsync() + : cmd.ExecuteReader(); - reader.Read(); + Assert.True(useAsync ? await reader.ReadAsync() : reader.Read()); // Read the original value back as a SqlDecimal, with matching scale and precision. SqlDecimal roundtrippedDecimal = reader.GetSqlDecimal(0); @@ -596,7 +699,7 @@ public static void TestScaledDecimalParameter_BulkCopy(string connectionString, // Synapse: Parse error at line: 2, column: 8: Incorrect syntax near 'TYPE'. // Enumeration is disabled to prevent generating empty test set when connection strings are not setup. - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] [MemberData(nameof(TestScaledDecimalParameter_Data), DisableDiscoveryEnumeration = true)] public static void TestScaledDecimalTVP_CommandSP(string connectionString, bool truncateScaledDecimal) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/SqlVariantParameterTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/SqlVariantParameterTests.cs index a6d5b741ed..546bd632ed 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/SqlVariantParameterTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/SqlVariantParameterTests.cs @@ -211,7 +211,7 @@ public void SqlType_BulkCopyFromDataTable_RoundTripsCorrectly(object paramValue, /// [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] [MemberData(nameof(BulkCopySqlTypeTestData), DisableDiscoveryEnumeration = true)] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public void SqlType_BulkCopyFromDataRow_RoundTripsCorrectly(object paramValue, string expectedTypeName, string expectedBaseTypeName) { // Arrange @@ -259,7 +259,7 @@ public void SqlType_BulkCopyFromDataRow_RoundTripsCorrectly(object paramValue, s /// /// Round trip sql_variant value using TVP with a SqlMetaData/SqlDataRecord source. /// - [Trait("Category", "flaky")] // Doesn't work well on Azure. + [Trait("category", "flaky")] // Doesn't work well on Azure. [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] [MemberData(nameof(SqlTypeTestData), DisableDiscoveryEnumeration = true)] public void SqlType_TvpFromSqlMetaData_RoundTripsCorrectly(object paramValue, string expectedTypeName, string expectedBaseTypeName) @@ -306,7 +306,7 @@ public void SqlType_TvpFromSqlMetaData_RoundTripsCorrectly(object paramValue, st /// /// Round trip sql_variant value using TVP with a SqlDataReader source. /// - [Trait("Category", "flaky")] // Doesn't work well on Azure. + [Trait("category", "flaky")] // Doesn't work well on Azure. [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] [MemberData(nameof(SqlTypeTestData), DisableDiscoveryEnumeration = true)] public void SqlType_TvpFromSqlDataReader_RoundTripsCorrectly(object paramValue, string expectedTypeName, string expectedBaseTypeName) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/StreamInputParameterTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/StreamInputParameterTests.cs index 5e8bfa5841..882cc98ab1 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/StreamInputParameterTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/StreamInputParameterTests.cs @@ -26,7 +26,7 @@ public StreamInputParameterTests() _connStr = DataTestUtility.TCPConnectionString; } - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] public void StreamInputParameterTest() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/TvpColumnBoundariesTests.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/TvpColumnBoundariesTests.cs index 81851902a5..a6d3c587db 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/TvpColumnBoundariesTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/TvpColumnBoundariesTests.cs @@ -19,7 +19,7 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests [Collection("ParameterBaselineTests")] public class TvpColumnBoundariesTests { - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] public void TvpColumnBoundariesTest() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/TvpTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/TvpTest.cs index a8baee786c..2373171a04 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/TvpTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/ParameterTest/TvpTest.cs @@ -295,26 +295,16 @@ private void TestTVPPermutations(SteStructuredTypeBoundaries bounds, bool runOnl continue; } - // Send list of SqlDataRecords as value - Console.WriteLine("------IEnumerable---------"); + // NOTE: The server objects created above (a table type and a stored procedure) must be + // dropped even when the body below throws. Their names are unique per permutation, + // so anything left behind stays in the shared test database forever. try { - param.Value = CreateListOfRecords(tvpPerm, baseValues); - ExecuteAndVerify(cmd, tvpPerm, baseValues, null); - } - catch (ArgumentException ae) - { - // some argument exceptions expected and should be swallowed - Console.WriteLine("Argument exception in value setup: {0}", ae.Message); - } - - if (!runOnlyDataRecordTest) - { - // send DbDataReader - Console.WriteLine("------DbDataReader---------"); + // Send list of SqlDataRecords as value + Console.WriteLine("------IEnumerable---------"); try { - param.Value = new TvpRestartableReader(CreateListOfRecords(tvpPerm, baseValues)); + param.Value = CreateListOfRecords(tvpPerm, baseValues); ExecuteAndVerify(cmd, tvpPerm, baseValues, null); } catch (ArgumentException ae) @@ -323,17 +313,35 @@ private void TestTVPPermutations(SteStructuredTypeBoundaries bounds, bool runOnl Console.WriteLine("Argument exception in value setup: {0}", ae.Message); } - // send datasets - Console.WriteLine("------DataTables---------"); - foreach (DataTable d in dtList) + if (!runOnlyDataRecordTest) { - param.Value = d; - ExecuteAndVerify(cmd, tvpPerm, null, d); + // send DbDataReader + Console.WriteLine("------DbDataReader---------"); + try + { + param.Value = new TvpRestartableReader(CreateListOfRecords(tvpPerm, baseValues)); + ExecuteAndVerify(cmd, tvpPerm, baseValues, null); + } + catch (ArgumentException ae) + { + // some argument exceptions expected and should be swallowed + Console.WriteLine("Argument exception in value setup: {0}", ae.Message); + } + + // send datasets + Console.WriteLine("------DataTables---------"); + foreach (DataTable d in dtList) + { + param.Value = d; + ExecuteAndVerify(cmd, tvpPerm, null, d); + } } } - - // And clean up - DropServerObjects(tvpPerm); + finally + { + // And clean up + DropServerObjects(tvpPerm); + } iter++; } @@ -892,11 +900,20 @@ private bool DoesRowMatchMetadata(object[] row, DataTable table) private void DropServerObjects(StePermutation tvpPerm) { - string dropText = "DROP PROC " + GetProcName(tvpPerm) + "; DROP TYPE " + GetTypeName(tvpPerm); using SqlConnection conn = new(_connStr); conn.Open(); - SqlCommand cmd = new(dropText, conn); + // NOTE: The procedure and the type are dropped by separate, individually guarded commands. + // Previously both drops shared a single batch, so when the procedure did not exist (the + // CREATE PROC step failed after CREATE TYPE succeeded) the batch aborted on the first + // statement and the table type was leaked into the shared test database. + DropServerObject(conn, "DROP PROC IF EXISTS " + GetProcName(tvpPerm)); + DropServerObject(conn, "DROP TYPE IF EXISTS " + GetTypeName(tvpPerm)); + } + + private static void DropServerObject(SqlConnection conn, string dropText) + { + using SqlCommand cmd = new(dropText, conn); try { cmd.ExecuteNonQuery(); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCommand/SqlCommandCancelTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCommand/SqlCommandCancelTest.cs index 830f1bb733..7be39757a2 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCommand/SqlCommandCancelTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCommand/SqlCommandCancelTest.cs @@ -329,7 +329,13 @@ private static void CancelAndDisposePreparedCommand(string constr) try { // Generate a query with a large number of results. - using (var command = new SqlCommand("select @P from sys.objects a cross join sys.objects b cross join sys.objects c cross join sys.objects d cross join sys.objects e cross join sys.objects f", connection)) + // The rows come from constant row sets rather than the system + // catalog, so the scan cannot contend with concurrent DDL on + // the shared test database. + const string rows = "(values(0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15))"; + string sql = $"select @P from {rows} a(n) cross join {rows} b(n) cross join {rows} c(n) " + + $"cross join {rows} d(n) cross join {rows} e(n) cross join {rows} f(n)"; + using (var command = new SqlCommand(sql, connection)) { command.Parameters.Add(new SqlParameter("@P", SqlDbType.Int) { Value = expectedValue }); connection.Open(); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCredentialTest/SqlCredentialTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCredentialTest/SqlCredentialTest.cs index 38295bd953..628ebfefe6 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCredentialTest/SqlCredentialTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlCredentialTest/SqlCredentialTest.cs @@ -198,18 +198,36 @@ private static void CreateTestUser(string username, string password) private static void DropTestUser(string username) { // Removes a created test user. - string dropUserCmd = $"IF EXISTS (SELECT * FROM sys.schemas WHERE name = '{username}') BEGIN DROP SCHEMA {username} END;" - + $"IF EXISTS (SELECT * FROM sys.database_principals WHERE type = 'S' AND name = '{username}') BEGIN DROP USER {username} END;" - + $"DROP LOGIN {username}"; + // NOTE: Each statement runs independently and is guarded so that a failure to drop the + // schema or database user does not leak the login, which is instance-wide and would + // otherwise survive well beyond the test run. + string[] dropStatements = + { + $"IF EXISTS (SELECT * FROM sys.schemas WHERE name = '{username}') BEGIN DROP SCHEMA {username} END;", + $"IF EXISTS (SELECT * FROM sys.database_principals WHERE type = 'S' AND name = '{username}') BEGIN DROP USER {username} END;", + $"IF EXISTS (SELECT * FROM sys.server_principals WHERE name = '{username}') BEGIN DROP LOGIN {username} END;", + }; // Pool must be cleared to prevent DROP LOGIN failure. SqlConnection.ClearAllPools(); using (var conn = new SqlConnection(DataTestUtility.TCPConnectionString)) - using (var cmd = new SqlCommand(dropUserCmd, conn)) + using (var cmd = new SqlCommand(string.Empty, conn)) { conn.Open(); - cmd.ExecuteNonQuery(); + + foreach (string statement in dropStatements) + { + try + { + cmd.CommandText = statement; + cmd.ExecuteNonQuery(); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(SqlCredentialTest)}: cleanup statement failed ({statement}): {ex.Message}"); + } + } } } } diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlNotificationTest/SqlNotificationTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlNotificationTest/SqlNotificationTest.cs index 3701b503a3..275a5b4bbf 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlNotificationTest/SqlNotificationTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/SqlNotificationTest/SqlNotificationTest.cs @@ -31,7 +31,19 @@ public SqlNotificationTest() _schemaQueue = $"[{_queueName}]"; - Setup(); + // NOTE: If setup fails part way through (for example the queue is created but the service + // is not), this constructor never returns, so xUnit never calls Dispose and the objects + // created so far would be leaked. Their names embed a GUID and xUnit creates one + // instance per test method, so those leaks accumulate quickly. + try + { + Setup(); + } + catch + { + Cleanup(); + throw; + } } public void Dispose() @@ -144,7 +156,7 @@ public void Test_SingleDependency_Stopped() } [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureServer))] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public void Test_SingleDependency_AllDefaults_SqlAuth() { Assert.True(SqlDependency.Start(_startConnectionString), "Failed to start listener."); @@ -310,10 +322,13 @@ private static string[] CreateSqlSetupStatements(string tableName, string queueN private static string[] CreateSqlCleanupStatements(string tableName, string queueName, string serviceName) { + // NOTE: Every statement is guarded so that cleanup is idempotent and so that a failure to + // remove one object does not leave the others (in particular the Service Broker queue + // and service) behind in the shared test database. return new string[] { - string.Format("DROP TABLE {0}", tableName), - string.Format("DROP SERVICE [{0}]", serviceName), - string.Format("DROP QUEUE {0}", queueName) + string.Format("IF (OBJECT_ID('{0}') IS NOT NULL) DROP TABLE {0}", tableName), + string.Format("IF EXISTS (SELECT 1 FROM sys.services WHERE name = '{0}') DROP SERVICE [{0}]", serviceName), + string.Format("IF (OBJECT_ID('{0}') IS NOT NULL) DROP QUEUE {0}", queueName) }; } @@ -324,7 +339,19 @@ private void Setup() private void Cleanup() { - RunSQL(CreateSqlCleanupStatements(_tableName, _schemaQueue, _serviceName)); + // Each statement is executed independently: one failure must not prevent the remaining + // objects from being dropped. + foreach (string statement in CreateSqlCleanupStatements(_tableName, _schemaQueue, _serviceName)) + { + try + { + RunSQL(statement); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(SqlNotificationTest)}: cleanup statement failed ({statement}): {ex.Message}"); + } + } } private int RunSQL(params string[] stmts) diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/TransactionTest/DistributedTransactionTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/TransactionTest/DistributedTransactionTest.cs index 7d1098ccad..c2db8c126b 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/TransactionTest/DistributedTransactionTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/TransactionTest/DistributedTransactionTest.cs @@ -51,7 +51,7 @@ public async Task Delegated_transaction_deadlock_in_SinglePhaseCommit() **/ private static bool s_EnlistedTransactionPreservedWhilePooledCondition => DataTestUtility.AreConnStringsSetup() && DataTestUtility.IsNotX86Architecture && DataTestUtility.IsNotManagedInstance(); - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(nameof(s_EnlistedTransactionPreservedWhilePooledCondition), Timeout = 10000)] public async Task Test_EnlistedTransactionPreservedWhilePooled() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/TransactionTest/TransactionTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/TransactionTest/TransactionTest.cs index 9aea760e56..91742465e3 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/TransactionTest/TransactionTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/TransactionTest/TransactionTest.cs @@ -174,11 +174,28 @@ private void DropTempTables() { using (var conn = new SqlConnection(_connectionString)) { - SqlCommand command = new SqlCommand( - string.Format("DROP TABLE [{0}]; DROP TABLE [{1}]", _tempTableName1, _tempTableName2), conn); conn.Open(); + + // NOTE: The drops are guarded and issued separately. Previously they shared a + // single unguarded batch, so a failure to drop the first table also leaked + // the second one. + DropTempTable(conn, _tempTableName1); + DropTempTable(conn, _tempTableName2); + } + } + + private static void DropTempTable(SqlConnection connection, string tableName) + { + try + { + using SqlCommand command = new SqlCommand( + string.Format("IF (OBJECT_ID('[{0}]') IS NOT NULL) DROP TABLE [{0}]", tableName), connection); command.ExecuteNonQuery(); } + catch (Exception ex) + { + Console.WriteLine($"TransactionTest: failed to drop table '{tableName}': {ex.Message}"); + } } public void ResetTables() diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/SqlServerTypesTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/SqlServerTypesTest.cs index 712d1ec2c6..1220f32fe2 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/SqlServerTypesTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/UdtTest/SqlServerTypesTest.cs @@ -37,7 +37,7 @@ public static class SqlServerTypesTest // Synapse: Parse error at line: 1, column: 48: Incorrect syntax near 'hierarchyid'. [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] #if NETFRAMEWORK - [Trait("Category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server + [Trait("category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server #endif public static void GetSchemaTableTest() { @@ -66,7 +66,7 @@ public static void GetSchemaTableTest() // Synapse: Parse error at line: 1, column: 48: Incorrect syntax near 'hierarchyid'. [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] #if NETFRAMEWORK - [Trait("Category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server + [Trait("category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server #endif public static void GetValueTest() { @@ -227,7 +227,7 @@ void ActAndAssert(int index, string expectedHexString) // Synapse: Parse error at line: 1, column: 41: Incorrect syntax near 'hierarchyid'. [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] #if NETFRAMEWORK - [Trait("Category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server + [Trait("category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server #endif public static void TestUdtSchemaMetadata() { @@ -384,7 +384,7 @@ private static string GetUdtName(Type udtClrType) // Synapse: Parse error at line: 1, column: 8: Incorrect syntax near 'geometry'. [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] #if NETFRAMEWORK - [Trait("Category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server + [Trait("category", "signed")] // Requires strong-name signed Microsoft.SqlServer.Server #endif public static void TestSqlServerTypesInsertAndRead() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs index 5c9970f6ed..a69a943e3d 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/NativeVectorTestsBase.cs @@ -110,11 +110,16 @@ public NativeVectorTestsBase() _connectionString = DataTestUtility.TCPConnectionString; _managementConnection = new SqlConnection(_connectionString); - _vectorTable = new Table(_managementConnection, "VectorTestTable", tableDefinition); - _bulkCopySourceTable = new Table(_managementConnection, "VectorBulkCopyTestTable", tableDefinition); - _vectorProcedure = new StoredProcedure(_managementConnection, - prefix: "VectorsAsVarcharSp", - definition: $@" + + // NOTE: If this constructor throws, xUnit never calls Dispose, so any object already created + // (each of which has a GUID-based name) would be left in the database permanently. + try + { + _vectorTable = new Table(_managementConnection, "VectorTestTable", tableDefinition); + _bulkCopySourceTable = new Table(_managementConnection, "VectorBulkCopyTestTable", tableDefinition); + _vectorProcedure = new StoredProcedure(_managementConnection, + prefix: "VectorsAsVarcharSp", + definition: $@" {VectorParameterName} vector({vectorDimensions}, {TestDataInstance.SqlServerTypeName}), -- Input: Serialized TElement[] as JSON string {VectorOutputParameterName} vector({vectorDimensions}, {TestDataInstance.SqlServerTypeName}) OUTPUT -- Output: Echoed back from latest inserted row AS @@ -130,6 +135,12 @@ public NativeVectorTestsBase() FROM {_vectorTable.Name} ORDER BY Id DESC; END;"); + } + catch + { + Dispose(); + throw; + } _selectCommand = $"SELECT {VectorColumnName} FROM {_vectorTable.Name} ORDER BY Id DESC"; _insertCommand = $"INSERT INTO {_vectorTable.Name} ({VectorColumnName}) VALUES ({VectorParameterName})"; @@ -150,15 +161,28 @@ protected virtual void Dispose(bool disposing) if (disposing) { - _vectorProcedure?.Dispose(); - _bulkCopySourceTable?.Dispose(); - _vectorTable?.Dispose(); + // Each drop is best-effort: failing to drop one object must not leak the others. + DisposeSafely(_vectorProcedure); + DisposeSafely(_bulkCopySourceTable); + DisposeSafely(_vectorTable); _managementConnection?.Dispose(); } _disposed = true; } + private static void DisposeSafely(IDisposable? disposable) + { + try + { + disposable?.Dispose(); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(NativeVectorTestsBase)}: cleanup failed: {ex.Message}"); + } + } + ~NativeVectorTestsBase() => Dispose(false); diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorBackwardCompatTestBase.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorBackwardCompatTestBase.cs index 1b2d4c792f..03298d3e37 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorBackwardCompatTestBase.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/VectorTest/VectorBackwardCompatTestBase.cs @@ -47,15 +47,22 @@ protected VectorBackwardCompatTestBase( { Output = output; _connection = new SqlConnection(s_connectionString); - _connection.Open(); - _vectorTable = new Table(_connection, namePrefix + "TestTable", - $"(Id INT PRIMARY KEY IDENTITY, VectorData {columnDefinition} NULL)"); + // NOTE: If this constructor throws, xUnit never calls Dispose, so the connection and the + // objects created so far (each with a GUID-based name) would be left behind permanently. + // Opening the connection is inside the try for the same reason: a failed Open still + // leaves a SqlConnection instance that nothing else will ever dispose. + try + { + _connection.Open(); + + _vectorTable = new Table(_connection, namePrefix + "TestTable", + $"(Id INT PRIMARY KEY IDENTITY, VectorData {columnDefinition} NULL)"); - _bulkCopySrcTable = new Table(_connection, namePrefix + "BulkCopyTestTable", - "(Id INT PRIMARY KEY IDENTITY, VectorData varchar(max) NULL)"); + _bulkCopySrcTable = new Table(_connection, namePrefix + "BulkCopyTestTable", + "(Id INT PRIMARY KEY IDENTITY, VectorData varchar(max) NULL)"); - string storedProcBody = $@" + string storedProcBody = $@" @InputVectorJson VARCHAR(MAX), -- Input: Serialized float[] as JSON string @OutputVectorJson VARCHAR(MAX) OUTPUT -- Output: Echoed back from latest inserted row AS @@ -72,7 +79,13 @@ @OutputVectorJson VARCHAR(MAX) OUTPUT -- Output: Echoed back from latest insert ORDER BY Id DESC; END;"; - _storedProc = new StoredProcedure(_connection, namePrefix + "AsVarcharSp", storedProcBody); + _storedProc = new StoredProcedure(_connection, namePrefix + "AsVarcharSp", storedProcBody); + } + catch + { + Dispose(); + throw; + } _selectCmdString = $"SELECT VectorData FROM {_vectorTable.Name} ORDER BY Id DESC"; _insertCmdString = $"INSERT INTO {_vectorTable.Name} (VectorData) VALUES (@VectorData)"; @@ -80,13 +93,32 @@ @OutputVectorJson VARCHAR(MAX) OUTPUT -- Output: Echoed back from latest insert public void Dispose() { - // RAII objects drop themselves on Dispose in reverse order. - _storedProc?.Dispose(); - _bulkCopySrcTable?.Dispose(); - _vectorTable?.Dispose(); + // RAII objects drop themselves on Dispose in reverse order. Each drop is best-effort so + // that one failure cannot leak the remaining objects. + DisposeSafely(_storedProc); + DisposeSafely(_bulkCopySrcTable); + DisposeSafely(_vectorTable); _connection?.Dispose(); } + // NOTE: This file is not in a nullable annotations context, so the parameter is annotated + // under a scoped `#nullable enable`. The nullability is not incidental: on the + // constructor-failure path Dispose runs with some of these fields still null, so accepting + // null is the contract rather than a defensive afterthought. +#nullable enable + private static void DisposeSafely(IDisposable? disposable) +#nullable restore + { + try + { + disposable?.Dispose(); + } + catch (Exception ex) + { + Console.WriteLine($"{nameof(VectorBackwardCompatTestBase)}: cleanup failed: {ex.Message}"); + } + } + #region Shared Helpers /// diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/WeakRefTestYukonSpecific/WeakRefTestYukonSpecific.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/WeakRefTestYukonSpecific/WeakRefTestYukonSpecific.cs index 864d1a5447..6b76e1f116 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/WeakRefTestYukonSpecific/WeakRefTestYukonSpecific.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/WeakRefTestYukonSpecific/WeakRefTestYukonSpecific.cs @@ -19,7 +19,7 @@ public static class WeakRefTestYukonSpecific // TODO Synapse: Remove dependency on Northwind database [ActiveIssue("6643", TestPlatforms.AnyUnix)] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] public static void TestReaderMars() { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/DiagnosticTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/DiagnosticTest.cs index eb64eecce8..440d16581f 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/DiagnosticTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/DiagnosticTest.cs @@ -23,7 +23,7 @@ namespace Microsoft.Data.SqlClient.ManualTesting.Tests { // TODO(ADO-39873): Re-enable these tests after addressing their flakiness. [Trait("Set", "3")] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] // Serialized execution: DiagnosticListener is global state, so these tests // must not run in parallel with each other. [Collection("DiagnosticTests")] diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/MetricsTest.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/MetricsTest.cs index 6d37051821..63eb1995a8 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/MetricsTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/TracingTests/MetricsTest.cs @@ -178,7 +178,7 @@ public void StasisCounters_Functional() Assert.Equal(0, SqlClientEventSourceProps.StasisConnections); } - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup), nameof(DataTestUtility.IsNotAzureSynapse))] public void TransactedConnectionPool_VerifyActiveConnectionCounters() { @@ -280,7 +280,7 @@ public void ReclaimedConnectionsCounter_Functional() } [ConditionalFact(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringsSetup))] - [Trait("Category", "flaky")] + [Trait("category", "flaky")] public void ConnectionPoolGroupsCounter_Functional() { SqlConnection.ClearAllPools(); diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolChurnRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolChurnRunner.cs index ca1930e451..d2b94cea14 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolChurnRunner.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolChurnRunner.cs @@ -20,6 +20,22 @@ namespace Microsoft.Data.SqlClient.PerformanceTests /// concern for the new ChannelDbConnectionPool, which aims to avoid extra allocations /// on the hot path (issue #3356). /// + /// This overlaps in shape — + /// the inner loop is identical — but not in purpose, and the two are not + /// interchangeable. Being single-threaded, this runner has no scheduling or wake-up + /// component, which makes it far more sensitive: its sync and async variants have + /// agreed to within 0.3 percentage points, whereas the concurrent runner's own + /// duplicate-workload parameter pairs have disagreed by around 20. Use this one to + /// decide whether per-checkout cost or allocation moved, and the concurrent runner to + /// decide whether hand-off between threads moved. A regression there with a flat result + /// here points at scheduling rather than at the checkout path. + /// + /// Adding a Parallelism=1 case to that runner would not replace this one. Its + /// [Params] are class-wide, so a single-threaded row would also be generated for + /// benchmarks it makes meaningless (MixedSyncAsyncContention would have no sync + /// worker left to mix in), and this runner's higher operation count and separate + /// iteration settings are what keep the signal clean. + /// /// The pool implementation (legacy vs V2) is a process-level choice — see the remarks /// on . Run twice (UseConnectionPoolV2 false /// then true) to compare. @@ -36,6 +52,27 @@ public class ConnectionPoolChurnRunner : BaseRunner [Params(1000)] public int OpsPerInvocation { get; set; } + /// + /// How many idle connections the pool holds while the single caller churns against it. + /// + /// + /// This is not a redundant axis, because the two pools order idle connections + /// differently: the legacy pool pops from a ConcurrentStack (LIFO) while the V2 + /// pool reads from an unbounded Channel (FIFO). At depth 1 that difference is + /// invisible, since both hand back the only connection there is. At a realistic depth + /// the legacy pool keeps returning the connection just released — one hot object — while + /// the V2 pool cycles through every idle connection in turn, touching all of their + /// buffers and parser state. Depth is therefore the axis that exposes reuse locality, + /// and measuring only depth 1 would hide it entirely. + /// + /// The middle depth is what makes the axis diagnostic rather than merely directional. + /// Depths 1 and 100 alone show only that reuse locality degrades somewhere in between; + /// they cannot distinguish a threshold (cost appears once the pool exceeds some working + /// set) from a gradient (cost grows smoothly with depth). Those imply different fixes. + /// + [Params(1, 10, 100)] + public int PoolDepth { get; set; } + private string _connectionString; [GlobalSetup] @@ -51,16 +88,48 @@ public void Setup() { Pooling = true, MaxPoolSize = 100, - // Pre-warm a single connection so the very first checkout is served from - // the pool rather than establishing a physical connection. - MinPoolSize = 1 + // Pin the floor at the requested depth so pruning cannot shrink the pool back + // down mid-run and change what the benchmark is measuring. + MinPoolSize = PoolDepth, + // Matches ConnectionPoolStressRunner. At the larger PoolDepth, setup establishes + // a hundred physical connections back to back, and the default 15s is tight for + // that against a loaded remote server. + ConnectTimeout = 60 }; _connectionString = builder.ConnectionString; - // Force the pool to exist and hold at least one idle connection. - using var warm = new SqlConnection(_connectionString); - warm.Open(); - warm.Close(); + PrewarmPool(); + } + + /// + /// Fills the pool with idle connections before measurement, so + /// every measured open is a pure checkout. + /// + /// + /// All connections are opened before any is released. Releasing as we go would let the + /// pool hand the same idle connection straight back and create only one, which would + /// silently collapse every depth to 1. MinPoolSize alone is not enough either: + /// both pools backfill it on a background task, so the first measured iteration would + /// otherwise still be racing that warm-up. + /// + private void PrewarmPool() + { + var warm = new SqlConnection[PoolDepth]; + try + { + for (int i = 0; i < warm.Length; i++) + { + warm[i] = new SqlConnection(_connectionString); + warm[i].Open(); + } + } + finally + { + foreach (var conn in warm) + { + conn?.Dispose(); + } + } } [GlobalCleanup] diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolContentionRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolContentionRunner.cs index 899d25f64c..2dbd6f94c8 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolContentionRunner.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolContentionRunner.cs @@ -30,6 +30,14 @@ namespace Microsoft.Data.SqlClient.PerformanceTests /// remarks on . Run twice (UseConnectionPoolV2 /// false then true) to compare. /// + /// Each sync workload is measured twice, once on threadpool threads + /// () and once on dedicated threads + /// (). Threadpool threads are the + /// realistic case, since sync database calls in ASP.NET run on them, and they are the + /// only configuration that can expose a waiter wake path which depends on the + /// threadpool having a free thread. Dedicated threads isolate the pool's own cost. The + /// pair separates a pool regression from a scheduling one. + /// /// Related issues: #601, #979, #3356 /// public class ConnectionPoolContentionRunner : BaseRunner @@ -41,13 +49,24 @@ public class ConnectionPoolContentionRunner : BaseRunner public int Parallelism { get; set; } /// - /// Maximum pool size, exercised across three regimes relative to - /// : larger than the worker count (idle spare - /// connections, no contention), equal to it (fully subscribed, no contention), and - /// smaller than it (pool exhaustion forces workers to wait for a connection to be - /// returned — back-pressure). + /// Maximum pool size, exercised across the full demand/capacity ladder relative to + /// : larger than the worker count (idle spare connections, no + /// contention), equal to it (fully subscribed, no contention), and progressively smaller + /// than it (pool exhaustion forces workers to wait for a connection to be returned — + /// back-pressure). /// - [Params(100, 50, 10)] + /// + /// At 50 these give demand/capacity ratios of 0.25, 0.5, 1, 2 + /// and 5. The intermediate over-subscribed step (ratio 2) is deliberate: back-pressure + /// regressions show up only once demand exceeds capacity, and without a point between + /// "fully subscribed" and "five times over" there is no way to tell how quickly the cost + /// grows once the pool starts running dry. + /// + /// 200 is included because it is the most commonly configured explicit Max Pool Size in + /// production, so the least-contended row corresponds to a real deployment rather than + /// only to the driver default. + /// + [Params(200, 100, 50, 25, 10)] public int MaxPoolSize { get; set; } /// @@ -114,6 +133,50 @@ public Task SteadyStateOpenQueryClose() return Task.WhenAll(tasks); } + /// + /// Same workload as , but driven by dedicated + /// threads instead of threadpool threads. + /// + /// Read the two together. A sync Open() that has to wait blocks whichever + /// thread it is running on. On threadpool threads that competes with the threadpool + /// itself, because a pool implementation whose waiter wake-up depends on a queued + /// continuation cannot make progress while every thread is blocked in a wait: the + /// wake-up is stuck behind thread injection. On the TFMs this project builds the + /// runtime notifies the pool of cooperative blocking and compensates quickly, so the + /// stall is tens to a few hundred milliseconds; that is the expectation measured here. + /// On net462 the Task wait never notifies the pool, so the wake-up waits on starvation + /// detection and hill climbing instead. That path is live, since the pool carries no + /// framework guards and the driver still ships net462, but this suite cannot measure it. + /// Dedicated threads remove that coupling, so this variant measures the pool's + /// intrinsic checkout/return cost with the scheduler taken out of the picture. + /// + /// A regression in both points at the pool itself. A regression only in the + /// threadpool variant points at the waiter wake path and shows up as tail latency + /// rather than a shifted median, so compare the distribution and not just the mean. + /// + [Benchmark] + public Task SteadyStateOpenQueryCloseDedicatedThreads() + { + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Factory.StartNew(() => + { + for (int op = 0; op < OpsPerWorker; op++) + { + using var conn = new SqlConnection(_connectionString); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT 1"; + _ = cmd.ExecuteScalar(); + // Dispose returns the connection to the pool. + } + }, TaskCreationOptions.LongRunning); + } + + return Task.WhenAll(tasks); + } + [Benchmark] public Task SteadyStateOpenQueryCloseAsync() { diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolRampRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolRampRunner.cs new file mode 100644 index 0000000000..344884fbc3 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolRampRunner.cs @@ -0,0 +1,177 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Data.SqlClient.PerformanceTests +{ + /// + /// Measures how quickly a cold pool can ramp up to N physical connections when N + /// callers arrive simultaneously and all of them need a connection at the same time. + /// + /// This is the workload the ChannelDbConnectionPool (V2) was designed for. The legacy + /// WaitHandleDbConnectionPool guards creation with a Semaphore(1, 1), so a cold + /// burst of N callers establishes physical connections one at a time: total latency is + /// roughly N x connect latency. V2 has no such gate, so the opens overlap and total + /// latency approaches a single connect. + /// + /// Contrast with , which also + /// starts from a cold pool but releases each connection immediately. Because nothing is + /// held, one physical connection can satisfy every caller in turn, so that benchmark + /// rewards a pool that grows as slowly as possible and penalizes concurrent creation. + /// Holding each connection until every caller has one removes that artifact: the pool + /// genuinely needs N connections, and the only variable left is how fast it can open + /// them. + /// + /// is always larger than so no + /// caller ever waits for a connection to be returned. Back-pressure on a saturated pool + /// is covered separately by . + /// + /// The pool implementation (legacy vs V2) is a process-level choice - see the remarks on + /// . Run twice (UseConnectionPoolV2 false then + /// true) to compare. + /// + /// Related issue: #3356 + /// + public class ConnectionPoolRampRunner : BaseRunner + { + /// + /// Number of callers that arrive simultaneously against a cold pool. Each one holds + /// its connection until all of them have connected, so the pool must open exactly + /// this many physical connections. + /// + [Params(10, 25, 50, 100)] + public int Parallelism { get; set; } + + /// + /// Max pool size. Deliberately larger than every value so + /// the ramp is never bounded by pool capacity. + /// + [Params(200)] + public int MaxPoolSize { get; set; } + + private string _connectionString; + + /// + /// Upper bound on the rendezvous wait. The rendezvous is released on the failure path + /// too, so this should never be reached; it exists so that a bug in the barrier fails + /// the run quickly instead of hanging the perf pipeline indefinitely. + /// + private static readonly TimeSpan s_rampTimeout = TimeSpan.FromMinutes(2); + + [GlobalSetup] + public void Setup() + { + Console.WriteLine( + "[ConnectionPoolRampRunner] Pool implementation: " + + (s_config.UseConnectionPoolV2 + ? "ChannelDbConnectionPool (V2)" + : "WaitHandleDbConnectionPool (legacy)")); + + var builder = new SqlConnectionStringBuilder(s_config.ConnectionString) + { + Pooling = true, + MaxPoolSize = MaxPoolSize, + // No pre-warming: every iteration must establish its own connections. + MinPoolSize = 0, + ConnectTimeout = 60 + }; + _connectionString = builder.ConnectionString; + } + + [IterationSetup] + public void IterationSetup() + { + // Start every iteration from a cold pool so the measurement is the ramp itself. + SqlConnection.ClearAllPools(); + } + + [GlobalCleanup] + public void Cleanup() => SqlConnection.ClearAllPools(); + + /// + /// Async cold-start ramp. All callers open concurrently and hold until the last one + /// has connected. + /// + [Benchmark] + public async Task ColdStartRampAsync() + { + using var allConnected = new CountdownEvent(Parallelism); + var release = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Run(async () => + { + using var conn = new SqlConnection(_connectionString); + try + { + await conn.OpenAsync(); + } + finally + { + // Signal from a finally so a caller that fails to open still counts as + // arrived. Otherwise the countdown never reaches zero, the release is + // never set, and every other caller awaits forever. + if (allConnected.Signal()) + { + release.TrySetResult(true); + } + } + + // Hold the connection until every caller has one, forcing the pool to + // grow to Parallelism physical connections. + await release.Task.WaitAsync(s_rampTimeout); + // Dispose returns the connection to the pool. + }); + } + + await Task.WhenAll(tasks); + } + + /// + /// Sync cold-start ramp. Uses dedicated threads rather than thread pool threads so + /// the measurement reflects pool ramp latency rather than thread pool injection + /// delay, which would otherwise dominate once the callers block. + /// + [Benchmark] + public void ColdStartRamp() + { + using var allConnected = new CountdownEvent(Parallelism); + + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Factory.StartNew(() => + { + using var conn = new SqlConnection(_connectionString); + try + { + conn.Open(); + } + finally + { + // See ColdStartRampAsync: signalling from a finally keeps a failed + // open from stranding every other caller in Wait(). + allConnected.Signal(); + } + + if (!allConnected.Wait(s_rampTimeout)) + { + throw new TimeoutException( + $"Cold-start ramp did not reach {Parallelism} connections within {s_rampTimeout}."); + } + // Dispose returns the connection to the pool. + }, TaskCreationOptions.LongRunning); + } + + Task.WaitAll(tasks); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolStressRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolStressRunner.cs index 9dad6f41d0..e65e08d22f 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolStressRunner.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolStressRunner.cs @@ -10,12 +10,20 @@ namespace Microsoft.Data.SqlClient.PerformanceTests { /// /// Stress-tests the connection pool with randomized parallel access patterns: - /// - Massive concurrent open/close churn + /// - Massive concurrent open/close churn, both sync and async /// - Randomized hold durations simulating real workloads /// - Mixed sync/async callers competing for pooled connections /// - Connection reuse with interleaved queries /// - Pool exhaustion and recovery under pressure /// + /// The pool is pre-warmed to full capacity in and is deliberately + /// not cleared between iterations, so these benchmarks measure steady-state + /// checkout and return rather than physical connection establishment. Establishing + /// connections costs orders of magnitude more than a pooled checkout (milliseconds versus + /// microseconds), so leaving any creation in the measured body swamps the pool cost this + /// class exists to measure. Cold-start behaviour is covered separately and deliberately by + /// . + /// /// Related issues: #601, #979, #3356 /// public class ConnectionPoolStressRunner : BaseRunner @@ -23,24 +31,59 @@ public class ConnectionPoolStressRunner : BaseRunner private string _connectionString; private string _tableName; + /// + /// Total checkouts performed by and + /// , held constant across every + /// and combination so a spread between cells reflects the pool, + /// not a different amount of work. Divides evenly by both Parallelism values. + /// + private const int RapidFireCheckouts = 1000; + /// /// Number of concurrent tasks hammering the pool. /// - [Params(10, 20, 25)] + /// + /// Two values spanning 5x rather than a tighter cluster. Every benchmark here runs at + /// every , so this axis is multiplied across seven benchmarks + /// and intermediate values buy far less than they cost — adjacent values in a narrow + /// range mostly re-measure the same regime. + /// + [Params(10, 50)] public int Parallelism { get; set; } /// /// Max pool size — controls how many physical connections the pool can hold. - /// When Parallelism exceeds this, tasks must wait for a free connection. /// + /// + /// The pool is pre-warmed to this many connections and pinned there by an equal + /// Min Pool Size, so this is the number of connections actually resident for the + /// whole run rather than just a ceiling. The limit itself is only reached by + /// , which deliberately oversubscribes it, and by the + /// fully-subscribed corner where equals this value. Everywhere + /// else the pool never saturates, so this parameter mostly varies how many idle + /// connections the checkout path is choosing among; treat a spread between two + /// MaxPoolSize values at the same Parallelism as a noise estimate rather than a real + /// effect. The fully-subscribed corner is the exception — there the spread is a real + /// effect, because one side has idle spares to choose from and the other has none. + /// + /// This reading holds for every benchmark here except + /// , whose task count is MaxPoolSize + + /// Parallelism: at Parallelism 10 the two cells run 60 and 110 tasks, so their spread + /// is mostly the extra work. That coupling is intrinsic, since exhausting the pool means + /// first saturating it. Elsewhere, do not make a benchmark body's operation count a + /// function of MaxPoolSize, or the spread stops being a noise estimate there too. + /// [Params(50, 100)] public int MaxPoolSize { get; set; } [GlobalSetup] public void Setup() { + // Pin Min Pool Size to Max Pool Size so the pool holds full capacity for the whole + // run: pruning cannot shrink it back down, and no benchmark body has to establish a + // physical connection. _connectionString = s_config.ConnectionString + - $";Pooling=True;Max Pool Size={MaxPoolSize};Min Pool Size=5;Connect Timeout=60"; + $";Pooling=True;Max Pool Size={MaxPoolSize};Min Pool Size={MaxPoolSize};Connect Timeout=60"; // Create a small table for query workloads. // Hash the machine name instead of using it verbatim: hostnames can be long enough @@ -48,22 +91,55 @@ public void Setup() // than using Math.Abs, which throws OverflowException when the hash is int.MinValue. string machineHash = ((uint)Environment.MachineName.GetHashCode()).ToString("x8"); _tableName = $"[perf_PoolStress_{machineHash}_{Guid.NewGuid():N}]"; - using var conn = new SqlConnection(_connectionString); - conn.Open(); - using var cmd = new SqlCommand( - $"CREATE TABLE {_tableName} (Id INT IDENTITY PRIMARY KEY, Val INT)", conn); - cmd.ExecuteNonQuery(); - // Seed a few rows so SELECT queries return data - using var insert = new SqlCommand( - $"INSERT INTO {_tableName} (Val) VALUES (1),(2),(3),(4),(5)", conn); - insert.ExecuteNonQuery(); + // Scoped so this connection is back in the pool before PrewarmPool runs: that method + // holds MaxPoolSize connections at once, so an extra live one would hit the cap and + // block until Connect Timeout. + { + using var conn = new SqlConnection(_connectionString); + conn.Open(); + using var cmd = new SqlCommand( + $"CREATE TABLE {_tableName} (Id INT IDENTITY PRIMARY KEY, Val INT)", conn); + cmd.ExecuteNonQuery(); + + // Seed a few rows so SELECT queries return data + using var insert = new SqlCommand( + $"INSERT INTO {_tableName} (Val) VALUES (1),(2),(3),(4),(5)", conn); + insert.ExecuteNonQuery(); + } + + PrewarmPool(); } - [IterationCleanup] - public void IterationCleanup() + /// + /// Fills the pool to live connections before any measurement + /// starts, so benchmark bodies only ever exercise checkout and return. + /// + /// + /// Every connection is opened before any is released. Releasing as we go would let the + /// pool hand the same idle connection back repeatedly and create only one, which is the + /// whole failure this is meant to avoid. Min Pool Size alone is not enough either: + /// it is backfilled lazily, so the first measured iteration would still pay for creation. + /// + private void PrewarmPool() { - SqlConnection.ClearAllPools(); + var warm = new SqlConnection[MaxPoolSize]; + try + { + for (int i = 0; i < warm.Length; i++) + { + warm[i] = new SqlConnection(_connectionString); + warm[i].Open(); + } + } + finally + { + // Returns them all to the pool; Min Pool Size keeps them resident from here on. + foreach (var conn in warm) + { + conn?.Dispose(); + } + } } [GlobalCleanup] @@ -79,14 +155,31 @@ public void Cleanup() /// /// Pure open/close churn — every task opens a pooled connection, immediately closes it, - /// and repeats. Measures raw pool checkout/return throughput under contention. - /// The per-task loop count scales with MaxPoolSize so total checkouts stay proportional - /// to pool capacity regardless of how the [Params] values change. + /// and repeats. With the pool pre-warmed to full capacity, every open is a pure checkout, + /// so this measures the pool's acquire/return path under concurrency. /// + /// + /// + /// Because nothing happens between checkout and return, the tasks stay phase-locked and + /// the idle channel oscillates around empty even though the pool is full. That makes this + /// benchmark unusually sensitive to how a returned connection is handed to a waiting + /// caller: under the V2 pool an inline TryRead miss parks the caller in + /// ReadAsync, so it resumes on a threadpool continuation. Read it as a + /// zero-hold-time worst case for wake-up scheduling, not as typical application + /// behaviour: real callers do some work while holding a connection, which decorrelates + /// returns from checkouts and lets the fast path hit. + /// + /// + /// Compare against for the same loop with no + /// concurrency, and for concurrency with a + /// realistic hold time. A regression here alongside flat or improved results in those two + /// indicates a change in wake-up scheduling rather than in checkout cost. + /// + /// [Benchmark] - public async Task RapidFireOpenClose() + public async Task RapidFireOpenCloseAsync() { - int iterationsPerTask = Math.Max(20, MaxPoolSize / Math.Max(1, Parallelism) * 4); + int iterationsPerTask = RapidFireCheckouts / Parallelism; var tasks = new Task[Parallelism]; for (int i = 0; i < Parallelism; i++) { @@ -103,6 +196,43 @@ public async Task RapidFireOpenClose() await Task.WhenAll(tasks); } + /// + /// Sync counterpart to : the same zero-hold churn, but + /// every checkout goes through the blocking Open() path. + /// + /// + /// Worth measuring separately because the two paths diverge inside the pool. The V2 pool + /// has no synchronous channel read, so a sync caller that misses the inline fast path has + /// to run the async wait synchronously, which is a materially different cost from + /// awaiting it. At 10 the pool has idle spares throughout, so + /// that cell measures concurrent sync checkout on the fast path. At Parallelism 50 with + /// 50 the pool is fully subscribed and the zero-hold loop can + /// drain the idle channel, so that cell is this class's coverage of the synchronous + /// wake path. + /// + /// Workers run on threadpool threads deliberately: sync database calls in ASP.NET run + /// there, so that is the configuration whose behaviour actually matters. + /// + [Benchmark] + public async Task RapidFireOpenCloseSync() + { + int iterationsPerTask = RapidFireCheckouts / Parallelism; + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Run(() => + { + for (int j = 0; j < iterationsPerTask; j++) + { + using var conn = new SqlConnection(_connectionString); + conn.Open(); + // immediate return to pool + } + }); + } + await Task.WhenAll(tasks); + } + /// /// Randomized hold — each task opens a connection, holds it for a random duration /// (0-50ms), optionally runs a query, then returns it. Simulates realistic mixed @@ -214,11 +344,18 @@ public async Task MultiCommandReuse() /// wait. Measures how well the pool handles back-pressure when all connections are /// checked out and callers are queued. /// + /// + /// sets how far the pool is oversubscribed, and so how deep + /// the queue of waiting callers gets. It is the queue depth rather than the concurrency + /// level here, because saturating the pool already takes + /// tasks before any caller has to wait at all. + /// [Benchmark] public async Task PoolExhaustionRecovery() { - // Ensure we exceed pool capacity - int taskCount = Math.Max(Parallelism, MaxPoolSize * 2); + // Saturate the pool, then oversubscribe it by Parallelism so exactly that many + // callers are queued waiting for a connection to come back. + int taskCount = MaxPoolSize + Parallelism; var tasks = new Task[taskCount]; for (int i = 0; i < taskCount; i++) { diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolThreadPoolPressureRunner.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolThreadPoolPressureRunner.cs new file mode 100644 index 0000000000..d898800da9 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/ConnectionPoolThreadPoolPressureRunner.cs @@ -0,0 +1,256 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.Data.SqlClient.PerformanceTests +{ + /// + /// Measures a saturated pool driven by sync callers on threadpool threads, with the + /// threadpool's minimum worker count pinned so the result is reproducible. + /// + /// A sync Open() against a saturated pool blocks its thread. When those threads + /// are threadpool threads, a pool whose waiter wake-up requires a queued continuation + /// cannot make progress: every thread is blocked in a wait, so the wake-up sits in the + /// queue until the threadpool injects another thread. On the TFMs this project builds the + /// runtime is told about cooperative blocking and compensates quickly, so stalls are tens + /// to a few hundred milliseconds. net462 has no equivalent notification and is slower, but + /// this suite does not build it, so that path is not measured here. + /// + /// covers the same shape at the default + /// threadpool floor, which makes it dependent on hill-climbing timing and therefore + /// noisy. Pinning the floor turns that into a controlled comparison: + /// + /// - below guarantees the + /// threadpool starts starved, so the wake path is exercised on every run. + /// - above pre-creates enough + /// threads that injection never gates progress. This is the control: a pool that only + /// regresses in the starved configuration has a wake-path problem, not a throughput + /// problem. + /// + /// The effect is tail latency, not a shifted median, so compare distributions rather + /// than means alone. + /// + /// A regression here is largely a statement about application configuration rather than a + /// pool defect, but the starved configuration is not exotic: the default minimum is + /// , which honours cgroup CPU quotas, so a service + /// in a 1-2 vCPU container runs with a floor of 1 or 2. This runner exists to characterise + /// where that boundary is and to catch it moving, not to drive the delta to zero. + /// + /// The pool implementation (legacy vs V2) is a process-level choice - see the remarks on + /// . Run twice (UseConnectionPoolV2 false then + /// true) to compare. + /// + /// Related issue: #3356 + /// + public class ConnectionPoolThreadPoolPressureRunner : BaseRunner + { + /// + /// Number of concurrent sync workers. Also the reference point for the non-starved + /// control in , so it is a constant rather than a + /// literal on the attribute. + /// + private const int WorkerParallelism = 50; + + /// + /// Number of concurrent sync workers, all running on threadpool threads. + /// + [Params(WorkerParallelism)] + public int Parallelism { get; set; } + + /// + /// Max pool size. Deliberately smaller than so most + /// workers must block waiting for a connection to be returned. Without that + /// back-pressure nobody waits and the wake path is never exercised. + /// + [Params(10)] + public int MaxPoolSize { get; set; } + + /// + /// Threadpool minimum worker thread count, pinned for the duration of the run. + /// + /// + /// Sourced from rather than fixed constants, + /// because this number is only meaningful relative to + /// — see that property for why. + /// + [ParamsSource(nameof(MinWorkerThreadsValues))] + public int MinWorkerThreads { get; set; } + + /// + /// The minimum worker counts to sweep, expressed as multiples of + /// . + /// + /// + /// Fixed constants would not survive a change of machine. The processor count is both + /// the value this parameter displaces (it is the runtime's own default floor) and the + /// size of the runtime's immediate cooperative-blocking injection budget, so the same + /// absolute number means "starved" on one host and "generous" on another. A constant + /// chosen to starve a 16-core benchmark machine would quietly stop starving anything on + /// a 4-core developer box, and the benchmark would keep reporting numbers that no longer + /// measure the wake path. + /// + /// The multiples map onto real deployments: a quarter of the processor count stands in + /// for the default floor of a 2-4 vCPU container, 1x is the runtime default that almost + /// every application actually runs, and 2x is the most common explicit multiplier in + /// shipped code. + /// + /// The control is the exception: it is pinned to twice + /// rather than a processor multiple, because a processor multiple is not guaranteed to + /// clear it. On a 4-core host ProcessorCount * 8 is 32, below the 50 workers, so + /// every case would be starved and the sweep would have no non-starved baseline to + /// compare against. Taking the larger of the two keeps the processor-relative scaling on + /// hosts where it is already sufficient. + /// + /// Floored at 1 because SetMinThreads rejects 0, and de-duplicated because the + /// lower multiples collapse together on very small hosts. + /// + public static IEnumerable MinWorkerThreadsValues => + new[] + { + Environment.ProcessorCount / 4, + Environment.ProcessorCount, + Environment.ProcessorCount * 2, + Math.Max(Environment.ProcessorCount * 8, WorkerParallelism * 2), + } + .Select(static value => Math.Max(1, value)) + .Distinct(); + + /// + /// Number of open/query/close operations each worker performs per invocation. + /// + [Params(20)] + public int OpsPerWorker { get; set; } + + private string _connectionString; + private int _originalMinWorkerThreads; + private int _originalMinCompletionPortThreads; + + [GlobalSetup] + public void Setup() + { + Console.WriteLine( + "[ConnectionPoolThreadPoolPressureRunner] Pool implementation: " + + (s_config.UseConnectionPoolV2 + ? "ChannelDbConnectionPool (V2)" + : "WaitHandleDbConnectionPool (legacy)")); + + ThreadPool.GetMinThreads( + out _originalMinWorkerThreads, out _originalMinCompletionPortThreads); + SetMinWorkerThreads(MinWorkerThreads, _originalMinCompletionPortThreads); + + var builder = new SqlConnectionStringBuilder(s_config.ConnectionString) + { + Pooling = true, + MaxPoolSize = MaxPoolSize, + MinPoolSize = 0 + }; + _connectionString = builder.ConnectionString; + } + + [GlobalCleanup] + public void Cleanup() + { + // Restoring matters beyond this benchmark: the floor is process-wide, so leaving it + // raised would silently change the conditions for every runner that follows in the + // same process. + SetMinWorkerThreads( + _originalMinWorkerThreads, _originalMinCompletionPortThreads); + } + + /// + /// Pins the threadpool worker floor, failing loudly if it does not take effect. + /// + /// is the only variable this benchmark manipulates, so a + /// refused or clamped request would leave every configuration running at the same floor + /// and produce a comparison that looks valid but measures nothing. The return value alone + /// is not sufficient evidence, so the value is also read back. + /// + private static void SetMinWorkerThreads(int workerThreads, int completionPortThreads) + { + if (!ThreadPool.SetMinThreads(workerThreads, completionPortThreads)) + { + throw new InvalidOperationException( + $"ThreadPool.SetMinThreads({workerThreads}, {completionPortThreads}) was refused. " + + "The threadpool floor is this benchmark's independent variable, so the run would " + + "otherwise report a comparison that did not actually vary it."); + } + + ThreadPool.GetMinThreads(out int actualWorkerThreads, out _); + if (actualWorkerThreads != workerThreads) + { + throw new InvalidOperationException( + $"ThreadPool.SetMinThreads({workerThreads}, ...) reported success but the floor " + + $"read back as {actualWorkerThreads}. The runtime clamped the request, so this " + + "configuration would not measure the intended threadpool pressure."); + } + } + + [IterationSetup] + public void IterationSetup() + { + // Warm the pool to MaxPoolSize so the measured run reflects steady-state + // checkout/return rather than first-time physical connection establishment. + WarmPool(MaxPoolSize); + } + + [IterationCleanup] + public void IterationCleanup() + { + using var conn = new SqlConnection(_connectionString); + SqlConnection.ClearPool(conn); + } + + [Benchmark] + public Task SaturatedSyncOpenOnThreadPool() + { + var tasks = new Task[Parallelism]; + for (int i = 0; i < Parallelism; i++) + { + tasks[i] = Task.Run(() => + { + for (int op = 0; op < OpsPerWorker; op++) + { + using var conn = new SqlConnection(_connectionString); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT 1"; + _ = cmd.ExecuteScalar(); + // Dispose returns the connection to the pool. + } + }); + } + + return Task.WhenAll(tasks); + } + + private void WarmPool(int count) + { + var conns = new SqlConnection[count]; + try + { + for (int i = 0; i < count; i++) + { + conns[i] = new SqlConnection(_connectionString); + conns[i].Open(); + } + } + finally + { + // Close and dispose every connection so they return to the pool and + // are not retained until GC (which would add allocation/GC noise). + for (int i = 0; i < count; i++) + { + conns[i]?.Close(); + conns[i]?.Dispose(); + } + } + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs index 7b136c6f2c..41d756c200 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs @@ -71,6 +71,8 @@ public class Benchmarks public RunnerJob ConnectionPoolStressRunnerConfig; public RunnerJob ConnectionPoolContentionRunnerConfig; public RunnerJob ConnectionPoolChurnRunnerConfig; + public RunnerJob ConnectionPoolRampRunnerConfig; + public RunnerJob ConnectionPoolThreadPoolPressureRunnerConfig; } public class RunnerJob diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs index 5a0d303e97..e326766f29 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/Program.cs @@ -55,6 +55,8 @@ public BenchmarkUnit(string name, Func selector, Type run new BenchmarkUnit("ConnectionPoolStress", b => b.ConnectionPoolStressRunnerConfig, typeof(ConnectionPoolStressRunner)), new BenchmarkUnit("ConnectionPoolContention", b => b.ConnectionPoolContentionRunnerConfig, typeof(ConnectionPoolContentionRunner)), new BenchmarkUnit("ConnectionPoolChurn", b => b.ConnectionPoolChurnRunnerConfig, typeof(ConnectionPoolChurnRunner)), + new BenchmarkUnit("ConnectionPoolRamp", b => b.ConnectionPoolRampRunnerConfig, typeof(ConnectionPoolRampRunner)), + new BenchmarkUnit("ConnectionPoolThreadPoolPressure", b => b.ConnectionPoolThreadPoolPressureRunnerConfig, typeof(ConnectionPoolThreadPoolPressureRunner)), }; /// diff --git a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc index 0ecc44e957..37bf73e960 100644 --- a/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc +++ b/src/Microsoft.Data.SqlClient/tests/PerformanceTests/runnerconfig.jsonc @@ -173,6 +173,24 @@ "InvocationCount": 1, "WarmupCount": 1, "RowCount": 0 + }, + "ConnectionPoolRampRunnerConfig": { + "Enabled": true, + "LaunchCount": 1, + "IterationCount": 15, + "InvocationCount": 1, + "WarmupCount": 1, + "RowCount": 0 + }, + // Measures tail latency from threadpool starvation, so it needs more iterations + // than the others: a stall shows up in the distribution, not in the median. + "ConnectionPoolThreadPoolPressureRunnerConfig": { + "Enabled": true, + "LaunchCount": 1, + "IterationCount": 25, + "InvocationCount": 1, + "WarmupCount": 1, + "RowCount": 0 } } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolTest.cs index d647ab0914..e9cb9adaa4 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolTest.cs @@ -235,6 +235,101 @@ out DbConnectionInternal? internalConnection Assert.Equal(pool.PoolGroupOptions.MaxPoolSize, pool.Count); } + /// + /// Verifies that an asynchronous request satisfied by an already-idle connection is + /// completed inline on the caller's thread, rather than being dispatched to the thread pool. + /// + /// + /// The fast path must report completion by returning true with the connection, exactly as + /// WaitHandleDbConnectionPool does on its inline hit. Completing the TaskCompletionSource + /// and returning false would look equivalent but is not: it sends + /// SqlConnection.InternalOpenAsync down its asynchronous branch, which allocates an + /// OpenAsyncRetry and schedules ContinueWith(..., TaskScheduler.Default), costing a thread + /// pool dispatch even though the result is already available. Asserting that the + /// TaskCompletionSource is left untouched is what pins that down. + /// + [Fact] + public void GetConnectionAsync_WithIdleConnection_ShouldCompleteInline() + { + // Arrange: take a connection and return it, leaving one connection idle in the pool. + var pool = ConstructPool(SuccessfulConnectionFactory); + SqlConnection owningConnection = new(); + + pool.TryGetConnection( + owningConnection, + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? pooledConnection + ); + Assert.NotNull(pooledConnection); + pool.ReturnInternalConnection(pooledConnection, owningConnection); + + // Act + TaskCompletionSource taskCompletionSource = new(); + SqlConnection secondOwner = new(); + var completed = pool.TryGetConnection( + secondOwner, + taskCompletionSource, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? internalConnection + ); + + // Assert: the request is reported as completed and the connection is handed back + // directly, matching WaitHandleDbConnectionPool's inline hit. This is what lets + // SqlConnection.InternalOpenAsync take its synchronous branch and skip the + // OpenAsyncRetry allocation and the ContinueWith thread pool dispatch. + Assert.True(completed); + Assert.Equal(pooledConnection, internalConnection); + + // The fast path must also activate the connection and assign ownership, exactly as the + // full path does. Without this the pool would hand back an unowned, unactivated + // connection and the assertions above would still pass. + Assert.Same(secondOwner, internalConnection!.Owner); + + // The TaskCompletionSource must be left alone; the caller abandons it on a + // synchronous completion. + Assert.False(taskCompletionSource.Task.IsCompleted); + + // The idle connection was reused rather than a second one being opened. + Assert.Equal(1, pool.Count); + } + + /// + /// Verifies that a synchronous request satisfied by an already-idle connection returns that + /// connection inline. + /// + [Fact] + public void GetConnection_WithIdleConnection_ShouldReturnInline() + { + // Arrange: take a connection and return it, leaving one connection idle in the pool. + var pool = ConstructPool(SuccessfulConnectionFactory); + SqlConnection owningConnection = new(); + + pool.TryGetConnection( + owningConnection, + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? pooledConnection + ); + Assert.NotNull(pooledConnection); + pool.ReturnInternalConnection(pooledConnection, owningConnection); + + // Act + SqlConnection secondOwner = new(); + var completed = pool.TryGetConnection( + secondOwner, + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? internalConnection + ); + + // Assert + Assert.True(completed); + Assert.Equal(pooledConnection, internalConnection); + Assert.Same(secondOwner, internalConnection!.Owner); + Assert.Equal(1, pool.Count); + } + /// /// Verifies that a waiting synchronous caller reuses a connection that is returned to an /// exhausted pool instead of creating a new physical connection. @@ -652,10 +747,19 @@ public void StressTestAsync() TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? internalConnection ); - internalConnection = await taskCompletionSource.Task; - pool.ReturnInternalConnection(internalConnection, owningObject); + + // A request satisfied from the pool's existing connections completes inline, + // returning the connection directly and leaving the TaskCompletionSource + // untouched. Only fall back to awaiting it when the request was handed off. + // This mirrors how the pool's callers consume TryGetConnection. + if (!completed) + { + internalConnection = await taskCompletionSource.Task; + } Assert.NotNull(internalConnection); + + pool.ReturnInternalConnection(internalConnection, owningObject); }); tasks.Add(t); } @@ -2282,6 +2386,94 @@ protected override DbConnectionInternal CreateConnection( #endregion + #region Saturated Sync Wait Tests + + /// + /// Verifies that the saturated synchronous checkout path makes forward progress when every + /// waiter is blocked in ReadChannelSyncOverAsync on a threadpool thread. This is the + /// net462 guard for that path, where the wake-up gets no cooperative-blocking notification; + /// it asserts liveness only, since the frameworks legitimately differ on latency. + /// + [Fact] + public void SyncCheckout_WhenSaturatedOnThreadPoolThreads_AllWaitersMakeProgress() + { + // Arrange + const int MaxPoolSize = 4; + ThreadPool.GetMinThreads(out int minWorker, out _); + + // More blocked waiters than the pool can serve AND than the threadpool floor, so the + // wake path is exercised rather than every worker simply getting its own thread. + // Deliberately uncapped: any ceiling would silently stop saturating on a host whose + // floor already exceeds it, turning this into a no-op exactly where it matters most. + int workerCount = minWorker + MaxPoolSize + 8; + + var poolGroupOptions = new DbConnectionPoolGroupOptions( + poolByIdentity: false, + minPoolSize: 0, + maxPoolSize: MaxPoolSize, + // Generous: a healthy run finishes far inside this, while a genuinely stuck wake + // path still fails the test rather than hanging the suite forever. + creationTimeout: 60, + loadBalanceTimeout: 0, + hasTransactionAffinity: true, + idleTimeout: 0); + var pool = ConstructPool(SuccessfulConnectionFactory, poolGroupOptions: poolGroupOptions); + + using var startGate = new ManualResetEventSlim(initialState: false); + var acquired = new ConcurrentBag(); + var failures = new ConcurrentBag(); + var workers = new Task[workerCount]; + + // Act: release every worker at once so they pile onto the pool together. + for (int i = 0; i < workerCount; i++) + { + workers[i] = Task.Factory.StartNew( + () => + { + try + { + startGate.Wait(); + bool got = pool.TryGetConnection( + new SqlConnection(), + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(60)), + out DbConnectionInternal? connection); + + acquired.Add(got && connection is not null); + + // Return promptly: with MaxPoolSize connections shared by every worker, + // each return is what wakes the next waiter. + if (connection is not null) + { + pool.ReturnInternalConnection(connection, connection.Owner); + } + } + catch (Exception ex) + { + failures.Add(ex); + } + }, + CancellationToken.None, + // LongRunning would hand each worker a dedicated thread, which is precisely the + // starvation this test needs to reproduce. Keep them on threadpool threads. + TaskCreationOptions.None, + TaskScheduler.Default); + } + + startGate.Set(); + + // Assert + Assert.True( + Task.WaitAll(workers, TimeSpan.FromSeconds(120)), + $"Saturated sync checkout did not drain: {acquired.Count} of {workerCount} workers " + + "finished. The waiter wake path is not making progress on this framework."); + Assert.Empty(failures); + Assert.Equal(workerCount, acquired.Count); + Assert.DoesNotContain(false, acquired); + } + + #endregion + #region Connection Timeout Awareness Tests /// @@ -2311,7 +2503,7 @@ protected override DbConnectionInternal CreateConnection( // The test running when the crash occurred: // Microsoft.Data.SqlClient.UnitTests.ConnectionPool.ChannelDbConnectionPoolTest.ConcurrentCallers_ShouldTimeoutIndependently // testhost_7576_20260724T235829_hangdump.dmp - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [Fact] public async Task ConcurrentCallers_ShouldTimeoutIndependently() { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/DbConnectionPoolAccessTokenTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/DbConnectionPoolAccessTokenTest.cs new file mode 100644 index 0000000000..c6e810cdd2 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/DbConnectionPoolAccessTokenTest.cs @@ -0,0 +1,484 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using System.Transactions; +using Microsoft.Data.ProviderBase; +using Microsoft.Data.SqlClient.ConnectionPool; +using Microsoft.Data.SqlClient.Tests.Common; +using Microsoft.SqlServer.TDS.PreLogin; +using Microsoft.SqlServer.TDS.Servers; +using Xunit; + +using static Microsoft.Data.SqlClient.UnitTests.ConnectionPool.PoolTestHarness; + +namespace Microsoft.Data.SqlClient.UnitTests.ConnectionPool +{ + /// + /// Verifies checkout-time token eviction separately from pool token-cache refresh. + /// The collection isolates pool-version switches; simulated logins need no Azure credentials. + /// + [Collection(SimulatedServerTestCollection.Name)] + public class DbConnectionPoolAccessTokenTest + { + private static readonly TimeSpan WaitTimeout = TimeSpan.FromSeconds(30); + + /// + /// Both pools defer token validation until checkout, reusing valid connections and replacing expired ones. + /// + [Theory] + [InlineData(PoolImplementation.WaitHandle, false, false)] + [InlineData(PoolImplementation.WaitHandle, false, true)] + [InlineData(PoolImplementation.WaitHandle, true, false)] + [InlineData(PoolImplementation.WaitHandle, true, true)] + [InlineData(PoolImplementation.Channel, false, false)] + [InlineData(PoolImplementation.Channel, false, true)] + [InlineData(PoolImplementation.Channel, true, false)] + [InlineData(PoolImplementation.Channel, true, true)] + public void Checkout_ValidatesIdleAccessToken(PoolImplementation implementation, bool async, bool expired) + { + using var fixture = new TokenPool(implementation); + using var owner = new SqlConnection(); + TokenConnection original = Request(fixture.Pool, owner, async); + original.Expired = expired; + int checks = original.ExpiryChecks; + fixture.Pool.ReturnInternalConnection(original, owner); + + // Like V1, returning a connection does not evaluate its token or refresh credentials. + Assert.Equal(checks, original.ExpiryChecks); + Assert.False(original.Disposed); + Assert.Equal(1, fixture.Pool.IdleCount); + + // This hits idle checkout. With one pool slot, eviction must free capacity for its replacement. + TokenConnection served = Request(fixture.Pool, owner, async); + Assert.Equal(!expired, ReferenceEquals(original, served)); + Assert.Equal(expired, original.Disposed); + Assert.True(original.ExpiryChecks > checks); + Assert.False(served.Expired); + Assert.Equal(1, fixture.Pool.Count); + fixture.Pool.ReturnInternalConnection(served, owner); + } + + /// + /// Freshly created connections must pass the expiry gate before activation, just like idle connections. + /// + [Theory] + [InlineData(PoolImplementation.WaitHandle, false)] + [InlineData(PoolImplementation.WaitHandle, true)] + [InlineData(PoolImplementation.Channel, false)] + [InlineData(PoolImplementation.Channel, true)] + public void Checkout_RejectsNewConnectionWithExpiredToken(PoolImplementation implementation, bool async) + { + // Only the first creation is expired, so retrying can succeed without waiting for time to pass. + using var fixture = new TokenPool(implementation, expireFirstCreation: true); + using var owner = new SqlConnection(); + TokenConnection served = Request(fixture.Pool, owner, async); + + Assert.Equal(2, fixture.Factory.Created.Count); + TokenConnection expired = fixture.Factory.Created[0]; + Assert.True(expired.Disposed); + // Activation would assign the connection to the caller; expiry must be rejected before then. + Assert.Equal(0, expired.Activations); + Assert.Same(fixture.Factory.Created[1], served); + Assert.Equal(1, fixture.Pool.Count); + fixture.Pool.ReturnInternalConnection(served, owner); + } + + /// + /// A direct channel handoff must validate expiry even though it bypasses the idle fast-path check. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task WaitingCheckout_RejectsExpiredToken(bool async) + { + using var fixture = new TokenPool(PoolImplementation.Channel); + var pool = (ChannelDbConnectionPool)fixture.Pool; + using var owner = new SqlConnection(); + using var waitingOwner = new SqlConnection(); + TokenConnection original = Request(pool, owner, async); + Task pending = Task.Run(() => Request(pool, waitingOwner, async)); + try + { + // Wait until the request is reading the channel, not merely scheduled on another thread. + // Returning the expired connection then exercises the post-wait gate, not idle lookup. + Assert.True(SpinWait.SpinUntil(() => pool.Reclaimer.ParkedWaiters == 1, WaitTimeout), + "The request did not reach the idle-channel wait."); + original.Expired = true; + pool.ReturnInternalConnection(original, owner); + + Assert.Same(pending, await Task.WhenAny(pending, Task.Delay(WaitTimeout))); + TokenConnection served = await pending; + Assert.NotSame(original, served); + Assert.True(original.Disposed); + Assert.False(served.Expired); + Assert.Equal(1, pool.Count); + pool.ReturnInternalConnection(served, waitingOwner); + } + finally + { + pool.Shutdown(); + Assert.Same(pending, await Task.WhenAny(pending, Task.Delay(WaitTimeout))); + } + } + + /// + /// Transaction affinity overrides expiry eviction until completion, avoiding disruption of the active transaction. + /// + [Theory] + [InlineData(PoolImplementation.WaitHandle, false)] + [InlineData(PoolImplementation.WaitHandle, true)] + [InlineData(PoolImplementation.Channel, false)] + [InlineData(PoolImplementation.Channel, true)] + public void TransactionCheckout_PreservesExpiredConnectionUntilTransactionEnds(PoolImplementation implementation, bool async) + { + using var fixture = new TokenPool(implementation); + using var owner = new SqlConnection(); + using var transaction = new CommittableTransaction(); + TokenConnection original; + using (var scope = new TransactionScope(transaction)) + { + original = Request(fixture.Pool, owner, async); + original.Expired = true; + int checks = original.ExpiryChecks; + fixture.Pool.ReturnInternalConnection(original, owner); + + // Return parks this connection in the transacted store, which takes precedence over idle reuse. + // The same transaction must get it back without checking expiry or breaking enlistment. + TokenConnection enlisted = Request(fixture.Pool, owner, async); + Assert.Same(original, enlisted); + Assert.Equal(checks, original.ExpiryChecks); + Assert.False(original.Disposed); + fixture.Pool.ReturnInternalConnection(enlisted, owner); + scope.Complete(); + } + transaction.Commit(); + + // Completion releases the connection to general circulation, where expiry eviction applies again. + TokenConnection served = Request(fixture.Pool, owner, async); + Assert.NotSame(original, served); + Assert.True(original.Disposed); + fixture.Pool.ReturnInternalConnection(served, owner); + } + + /// + /// Physical-connection expiry triggers replacement; only an expiring cached token requires another callback. + /// + [Theory] + [InlineData(false, false, -1, false)] + [InlineData(false, true, -1, false)] + [InlineData(true, false, -1, false)] + [InlineData(true, true, -1, false)] + [InlineData(false, false, 300, false)] + [InlineData(false, true, 300, false)] + [InlineData(true, false, 300, false)] + [InlineData(true, true, 300, false)] + [InlineData(false, false, -1, true)] + [InlineData(false, true, -1, true)] + [InlineData(true, false, -1, true)] + [InlineData(true, true, -1, true)] + [InlineData(false, false, 300, true)] + [InlineData(false, true, 300, true)] + [InlineData(true, false, 300, true)] + [InlineData(true, true, 300, true)] + public async Task AccessTokenCallback_CheckoutRejectsExpiredToken(bool usePoolV2, bool async, int expiresInSeconds, bool expireCachedToken) + { + using var poolVersion = new ConnectionPoolVersionScope(usePoolV2); + using var server = new TdsServer(new TdsServerArguments + { + FedAuthRequiredPreLoginOption = TdsPreLoginFedAuthRequiredOption.FedAuthRequired + }); + server.Start(); + var builder = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + MaxPoolSize = 1, + ConnectTimeout = 600, + Enlist = false + }; + int callbackInvocations = 0; + using var connection = new SqlConnection(builder.ConnectionString) + { + AccessTokenCallback = (_, _) => + { + Interlocked.Increment(ref callbackInvocations); + return Task.FromResult(new SqlAuthenticationToken("invalid", DateTimeOffset.UtcNow.AddHours(2))); + } + }; + + try + { + // The first physical login populates the token cache; ordinary reopen only reuses the socket. + await OpenConnection(connection, async); + var original = Assert.IsType(connection.InnerConnection); + Assert.False(original.IsAccessTokenExpired); + Assert.Equal(1, callbackInvocations); + connection.Close(); + await OpenConnection(connection, async); + Assert.Same(original, connection.InnerConnection); + Assert.Equal(1, callbackInvocations); + + // Change metadata, not wall-clock time: -1 is expired and 300 is within the 600-second buffer. + // The physical connection and the pool cache hold separate expiry values. + FieldInfo? tokenField = original.GetType().GetField("_fedAuthToken", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(tokenField); + DateTimeOffset expiry = DateTimeOffset.UtcNow.AddSeconds(expiresInSeconds); + tokenField!.SetValue(original, new SqlFedAuthToken(new SqlAuthenticationToken("invalid", expiry))); + Assert.Equal(expiresInSeconds > 0, expiry > DateTimeOffset.UtcNow); + Assert.True(original.IsAccessTokenExpired); + IDbConnectionPool pool = original.Pool; + var cachedToken = Assert.Single(pool.AuthenticationContexts); + if (expireCachedToken) + { + // Leaving the cache fresh models another login already having refreshed the pool's token. + // Aging it instead requires the replacement login to invoke the callback. + pool.AuthenticationContexts[cachedToken.Key] = new DbConnectionPoolAuthenticationContext( + cachedToken.Value.AccessToken, expiry.UtcDateTime); + } + connection.Close(); + Assert.Equal(1, callbackInvocations); + + // Checkout evicts the old physical connection. Its replacement either uses the fresh cache + // or invokes the callback and updates the cache; eviction alone must not force token acquisition. + await OpenConnection(connection, async); + Assert.NotSame(original, connection.InnerConnection); + Assert.False(connection.InnerConnection.IsAccessTokenExpired); + Assert.Equal(expireCachedToken ? 2 : 1, callbackInvocations); + Assert.True(pool.AuthenticationContexts[cachedToken.Key].ExpirationTime > expiry.UtcDateTime); + + // Once replaced, reuse must neither create another physical connection nor invoke the callback. + DbConnectionInternal replacement = connection.InnerConnection; + connection.Close(); + await OpenConnection(connection, async); + Assert.Same(replacement, connection.InnerConnection); + Assert.Equal(expireCachedToken ? 2 : 1, callbackInvocations); + } + finally + { + SqlConnection.ClearPool(connection); + } + } + + /// + /// New physical logins refresh expired or nearly expired cached tokens but reuse sufficiently valid ones. + /// + [Theory] + [InlineData(false, false, -1)] + [InlineData(false, true, -1)] + [InlineData(true, false, -1)] + [InlineData(true, true, -1)] + [InlineData(false, false, 300)] + [InlineData(false, true, 300)] + [InlineData(true, false, 300)] + [InlineData(true, true, 300)] + [InlineData(false, false, 3600)] + [InlineData(false, true, 3600)] + [InlineData(true, false, 3600)] + [InlineData(true, true, 3600)] + public async Task AccessTokenCallback_NewPhysicalConnectionHonorsCachedTokenExpiry(bool usePoolV2, bool async, int expiresInSeconds) + { + using var poolVersion = new ConnectionPoolVersionScope(usePoolV2); + using var server = new TdsServer(new TdsServerArguments + { + FedAuthRequiredPreLoginOption = TdsPreLoginFedAuthRequiredOption.FedAuthRequired + }); + server.Start(); + var builder = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + MaxPoolSize = 2, + Enlist = false + }; + int callbackInvocations = 0; + using var first = new SqlConnection(builder.ConnectionString) + { + AccessTokenCallback = (_, _) => + { + int invocation = Interlocked.Increment(ref callbackInvocations); + return Task.FromResult(new SqlAuthenticationToken($"invalid-{invocation}", DateTimeOffset.UtcNow.AddHours(2))); + } + }; + // The callback delegate is part of the pool key; share it to exercise the same token cache. + using var second = new SqlConnection(builder.ConnectionString) + { + AccessTokenCallback = first.AccessTokenCallback + }; + + try + { + await OpenConnection(first, async); + Assert.Equal(1, callbackInvocations); + IDbConnectionPool pool = first.InnerConnection.Pool; + var entry = Assert.Single(pool.AuthenticationContexts); + var cachedToken = new DbConnectionPoolAuthenticationContext( + entry.Value.AccessToken, DateTime.UtcNow.AddSeconds(expiresInSeconds)); + pool.AuthenticationContexts[entry.Key] = cachedToken; + + // Keep the first connection checked out so the second must perform a physical login. + await OpenConnection(second, async); + Assert.NotSame(first.InnerConnection, second.InnerConnection); + Assert.Same(pool, second.InnerConnection.Pool); + Assert.False(second.InnerConnection.IsAccessTokenExpired); + // -1 and 300 seconds require refresh (the cache's 10-minute window); 3600 seconds + // is beyond even its 45-minute opportunistic refresh window, so it must reuse the token. + bool refreshed = expiresInSeconds <= 600; + Assert.Equal(refreshed ? 2 : 1, callbackInvocations); + DbConnectionPoolAuthenticationContext current = pool.AuthenticationContexts[entry.Key]; + if (refreshed) + { + Assert.NotSame(cachedToken, current); + Assert.NotEqual(cachedToken.AccessToken, current.AccessToken); + Assert.True(current.ExpirationTime > cachedToken.ExpirationTime); + } + else + { + Assert.Same(cachedToken, current); + } + + // Physical reuse skips authentication entirely, regardless of which cache branch ran above. + DbConnectionInternal reused = second.InnerConnection; + second.Close(); + await OpenConnection(second, async); + Assert.Same(reused, second.InnerConnection); + Assert.Equal(refreshed ? 2 : 1, callbackInvocations); + } + finally + { + SqlConnection.ClearPool(first); + } + } + + /// + /// Exercises the selected public open API, bounding asynchronous opens with cancellation. + /// + /// Connection to open against the simulated server. + /// Whether to use OpenAsync instead of Open. + /// A task completing when the connection is open. + private static async Task OpenConnection(SqlConnection connection, bool async) + { + if (async) + { + using var cancellation = new CancellationTokenSource(WaitTimeout); + await connection.OpenAsync(cancellation.Token); + } + else + { + connection.Open(); + } + } + + /// + /// Requests a stub connection directly from a pool, handling inline and deferred completion. + /// + /// Pool under test. + /// Owner passed to connection activation. + /// Whether to supply the completion source used by asynchronous opens. + /// The connection assigned to the owner. + private static TokenConnection Request(IDbConnectionPool pool, SqlConnection owner, bool async) + { + // Async opens carry the ambient transaction in AsyncState so worker threads preserve affinity. + TaskCompletionSource? completion = async + ? new TaskCompletionSource(Transaction.Current, TaskCreationOptions.RunContinuationsAsynchronously) + : null; + bool completed = pool.TryGetConnection(owner, completion, TimeoutTimer.StartNew(WaitTimeout), out DbConnectionInternal? connection); + if (!completed) + { + Assert.NotNull(completion); + Assert.True(completion!.Task.Wait(WaitTimeout), "The connection request did not complete."); + connection = completion.Task.GetAwaiter().GetResult(); + } + return Assert.IsType(connection); + } + + /// + /// Uses one pool slot and the harness's frozen clock to isolate checkout decisions from maintenance. + /// + private sealed class TokenPool : IDisposable + { + internal TokenFactory Factory { get; } + internal IDbConnectionPool Pool { get; } + + /// Creates an isolated pool backed by controllable token connections. + /// Pool implementation to exercise. + /// Whether the first created connection starts expired. + internal TokenPool(PoolImplementation implementation, bool expireFirstCreation = false) + { + Factory = new TokenFactory(expireFirstCreation); + Pool = ConstructPool(implementation, Factory, maxPoolSize: 1, creationTimeout: 30000); + } + + /// Stops maintenance and disposes idle connections and any left checked out by a failed test. + public void Dispose() + { + Pool.Shutdown(); + Pool.Clear(); + foreach (TokenConnection connection in Factory.Created) + { + if (!connection.Disposed) + { + connection.Dispose(); + } + } + } + } + + /// + /// Records physical creations and can expire the first one to force a checkout retry. + /// + private sealed class TokenFactory(bool expireFirstCreation) : SqlConnectionFactory + { + internal List Created { get; } = new(); + + /// + protected override DbConnectionInternal CreateConnection(SqlConnectionOptions options, ConnectionPoolKey poolKey, + DbConnectionPoolGroupProviderInfo poolGroupProviderInfo, IDbConnectionPool pool, DbConnection owningConnection, TimeoutTimer timeout) + { + var connection = new TokenConnection { Expired = expireFirstCreation && Created.Count == 0 }; + Created.Add(connection); + return connection; + } + } + + /// + /// Makes expiry deterministic and records whether the pool checks, activates, or disposes the connection. + /// + private sealed class TokenConnection : ChannelDbConnectionPoolTest.StubDbConnectionInternal + { + internal bool Expired { get; set; } + internal int ExpiryChecks { get; private set; } + internal int Activations { get; private set; } + internal bool Disposed { get; private set; } + + internal override bool IsAccessTokenExpired + { + get + { + ExpiryChecks++; + return Expired; + } + } + + /// + protected override void Activate(Transaction transaction) + { + Activations++; + base.Activate(transaction); + } + + /// + public override void Dispose() + { + Disposed = true; + base.Dispose(); + } + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs index c5d628c618..163023a0f4 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs @@ -130,6 +130,34 @@ public void TryGetConnection_WhenFactoryThrows_EntersBlockingPeriod() Assert.Equal(1, factory.CreateConnectionCallCount); } + /// + /// Shutdown preserves the cached error for admitted waiters and lets its timer expire. + /// + [Fact] + public void Shutdown_WhileBlocked_PreservesErrorUntilExpiry() + { + // Arrange + var clock = new FakeTimeProvider(); + SqlException failure = SqlExceptionHelper.CreateSqlException("server unreachable"); + var factory = new ConfigurableSqlConnectionFactory(_ => throw failure); + var pool = CreatePool(factory, timeProvider: clock); + using var owner = new SqlConnection(); + + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + + // Act + pool.Shutdown(); + + // Assert + Assert.True(pool.ErrorOccurred); + + // Act: expire the preserved blocking period. + clock.Advance(TimeSpan.FromSeconds(5)); + + // Assert + Assert.False(pool.ErrorOccurred); + } + /// /// Verifies that once the pool is in the blocking period, a subsequent request fast-fails /// with the cached exception without invoking the connection factory again. The first throw diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolShutdownTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolShutdownTest.cs index 4ec6370d88..c0be93f156 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolShutdownTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolShutdownTest.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Data.Common; using System.Threading; using System.Threading.Tasks; using Microsoft.Data.Common.ConnectionString; @@ -17,7 +18,7 @@ namespace Microsoft.Data.SqlClient.UnitTests.ConnectionPool /// public class WaitHandleDbConnectionPoolShutdownTest { - private static WaitHandleDbConnectionPool CreatePool(int maxPoolSize = 5) + private static WaitHandleDbConnectionPool CreatePool(int maxPoolSize = 5, SqlConnectionFactory? factory = null) { var poolGroupOptions = new DbConnectionPoolGroupOptions( poolByIdentity: false, @@ -34,7 +35,7 @@ private static WaitHandleDbConnectionPool CreatePool(int maxPoolSize = 5) poolGroupOptions); var pool = new WaitHandleDbConnectionPool( - new WaitHandleDbConnectionPoolTransactionTest.MockSqlConnectionFactory(), + factory ?? new WaitHandleDbConnectionPoolTransactionTest.MockSqlConnectionFactory(), dbConnectionPoolGroup, DbConnectionPoolIdentity.NoIdentity, new DbConnectionPoolProviderInfo()); @@ -67,13 +68,17 @@ public void Shutdown_DisposesCleanupTimer() Assert.Null(pool._cleanupTimer); } - // Drains idle stacks. + /// + /// Leaves idle connections for the factory's explicit or deferred Clear call. + /// [Fact] - public void Shutdown_DrainsIdleStacks() + public void Shutdown_LeavesIdleConnectionsUntilClear() { + // Arrange var pool = CreatePool(); - // Vend a few connections then return them so they sit in _stackNew. + // An empty pool would pass even if Shutdown still called Clear internally. + // Keep idle inventory so the two lifecycle operations have distinguishable effects. var owner1 = new SqlConnection(); var owner2 = new SqlConnection(); pool.TryGetConnection(owner1, taskCompletionSource: null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? c1); @@ -86,10 +91,32 @@ public void Shutdown_DrainsIdleStacks() Assert.Equal(2, pool.IdleCount); Assert.Equal(2, pool.Count); - pool.Shutdown(); - - Assert.Equal(0, pool.IdleCount); - Assert.Equal(0, pool.Count); + try + { + // Act + pool.Shutdown(); + + // Assert: unchanged inventory and poolability detect both effects of an + // unintended Clear: draining idle objects and marking them non-poolable. + Assert.False(pool.IsRunning); + Assert.Equal(2, pool.IdleCount); + Assert.Equal(2, pool.Count); + Assert.True(c1!.CanBePooled); + Assert.True(c2!.CanBePooled); + + // Act: perform the Clear that the factory owns, separately from shutdown. + pool.Clear(); + + // Assert + Assert.Equal(0, pool.IdleCount); + Assert.Equal(0, pool.Count); + } + finally + { + // Cleanup + pool.Shutdown(); + pool.Clear(); + } } // Shutdown is idempotent. @@ -160,70 +187,151 @@ public void TryGetConnection_Async_AfterShutdown_ShortCircuits_NoPendingOpenSche Assert.Equal(0, Volatile.Read(ref pool._waitCount)); } - // Shutdown wakes up a thread parked in WaitHandle.WaitAny. - [Trait("Category", "flaky")] - // Failed Microsoft.Data.SqlClient.UnitTests.ConnectionPool.WaitHandleDbConnectionPoolShutdownTest.Shutdown_UnblocksSyncWaiter [5 s] - // ##[error]EXEC(0,0): Error Message: - // EXEC : error Message: [D:\a\_work\1\s\build.proj] - // Waiter did not park within 5s. - // Stack Trace: - // at Microsoft.Data.SqlClient.UnitTests.ConnectionPool.WaitHandleDbConnectionPoolShutdownTest.Shutdown_UnblocksSyncWaiter() in D:\a\_work\1\s\src\Microsoft.Data.SqlClient\tests\UnitTests\ConnectionPool\WaitHandleDbConnectionPoolShutdownTest.cs:line 207 - [Fact] - public void Shutdown_UnblocksSyncWaiter() + /// + /// Starts with no idle connections. Pauses the first physical connection creation + /// while it holds the creation semaphore, then admits a second request that waits + /// for that semaphore. Shuts down the pool before releasing the first creation. + /// Both requests create their own connections on the retired pool, and both + /// connections are destroyed when returned. In the cancellation case, the worker + /// returns and destroys the second connection instead of delivering it to the caller. + /// + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Shutdown_InFlightRequest_CompletesOnRetiredPool(bool async, bool cancel) { - var pool = CreatePool(maxPoolSize: 1); + // Arrange + // Initiate a request to the pool. It will be blocked by the gated connection factory. + using var factory = new GatedConnectionFactory(); + var pool = CreatePool(maxPoolSize: 2, factory: factory); + using var firstOwner = new SqlConnection(); + using var pendingOwner = new SqlConnection(); + var completion = new TaskCompletionSource(); + Task first = Acquire(pool, firstOwner, completion: null); + Task? pending = null; + + // Make sure the first request is blocked in the connection factory. Then, initiate a second request. + // The second request will be blocked on the create semaphore in the pool. + Assert.True(factory.Entered.Wait(TimeSpan.FromSeconds(10))); + pending = Acquire(pool, pendingOwner, async ? completion : null); + // Make sure the second request is also blocked. + Assert.True(SpinWait.SpinUntil(() => Volatile.Read(ref pool._waitCount) == 2, TimeSpan.FromSeconds(10))); + Assert.Equal(1, factory.CreateCount); + Assert.False(first.IsCompleted); + Assert.False(pending.IsCompleted); + + // Act + // Now, shut down the pool. New requests will no longer be accepted, but in-flight requests should proceed. + pool.Shutdown(); + if (cancel) + { + completion.SetCanceled(); + } - // Saturate the pool. - var owner = new SqlConnection(); - Assert.True(pool.TryGetConnection(owner, taskCompletionSource: null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? blocking)); - Assert.NotNull(blocking); + // Unblock the first request, allowing both to proceed, in turn. + factory.Release.Set(); - // Park a sync waiter on a worker thread with a long creation timeout. - DbConnectionInternal? waiterResult = null; - bool waiterCompleted = false; - Exception? waiterEx = null; + // Assert + // Wait for the first request to complete + Assert.False(pool.IsRunning); + Assert.Same(first, await Task.WhenAny(first, Task.Delay(TimeSpan.FromSeconds(10)))); + DbConnectionInternal? firstConnection = await first; + Assert.NotNull(firstConnection); + Assert.Same(pool, firstConnection.Pool); + + // Wait for the second request to complete + Assert.Same(pending, await Task.WhenAny(pending, Task.Delay(TimeSpan.FromSeconds(10)))); + if (cancel) + { + await Assert.ThrowsAnyAsync(() => pending); + Assert.True(SpinWait.SpinUntil(() => pool.Count == 1 && Volatile.Read(ref pool._waitCount) == 0, TimeSpan.FromSeconds(10))); + } + else + { + DbConnectionInternal? pendingConnection = await pending; + Assert.NotNull(pendingConnection); + Assert.Same(pool, pendingConnection.Pool); + Assert.Equal(0, Volatile.Read(ref pool._waitCount)); + } + + // Assert that both requests created new connections + Assert.Equal(2, factory.CreateCount); - var t = new Thread(() => + // Cleanup + factory.Release.Set(); + pool.Shutdown(); + await ReturnWhenCompleted(pool, firstOwner, first); + if (pending is not null) { - try - { - waiterCompleted = pool.TryGetConnection( - new SqlConnection(), - taskCompletionSource: null, - TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), - out waiterResult); - } - catch (Exception ex) + await ReturnWhenCompleted(pool, pendingOwner, pending); + } + + // Assert: returned connections were destroyed rather than pooled. + Assert.Equal(0, pool.IdleCount); + Assert.Equal(0, pool.Count); + } + + /// Starts a sync acquisition on a dedicated thread or queues an async acquisition. + private static Task Acquire(WaitHandleDbConnectionPool pool, SqlConnection owner, + TaskCompletionSource? completion) + { + TimeoutTimer timer = TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)); + if (completion is null) + { + return Task.Factory.StartNew(() => { - waiterEx = ex; - } - }) - { IsBackground = true }; - t.Start(); - - // Wait deterministically until the worker has incremented _waitCount, which - // happens immediately before it enters WaitHandle.WaitAny. Polling avoids the - // CI-flakiness of a fixed Thread.Sleep on slow agents. Volatile.Read ensures - // we see the worker's Interlocked.Increment without depending on CPU memory - // ordering of plain int reads. - var deadline = DateTime.UtcNow.AddSeconds(5); - while (DateTime.UtcNow < deadline && Volatile.Read(ref pool._waitCount) < 1) + Assert.True(pool.TryGetConnection(owner, null, timer, out DbConnectionInternal connection)); + return (DbConnectionInternal?)connection; + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + } + // A false return here means async acquisition was queued, not that it failed. + // Its eventual result is delivered through completion.Task. + Assert.False(pool.TryGetConnection(owner, completion, timer, out DbConnectionInternal pending)); + Assert.Null(pending); + return completion.Task!; + } + + /// Drains test work and returns successful acquisitions even when an assertion failed. + private static async Task ReturnWhenCompleted(WaitHandleDbConnectionPool pool, SqlConnection owner, Task task) + { + Assert.Same(task, await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(20)))); + if (task.Status == TaskStatus.RanToCompletion && task.Result is { } connection) + { + pool.ReturnInternalConnection(connection, owner); + } + else if (task.IsFaulted) { - Thread.Yield(); + // Observe failures during assertion cleanup without replacing the original failure. + _ = task.Exception; } - Assert.True(Volatile.Read(ref pool._waitCount) >= 1, "Waiter did not park within 5s."); - Assert.True(t.IsAlive, "Waiter should be parked, but thread already exited."); + } - pool.Shutdown(); + /// Holds the first physical creation so a second acquisition waits on its semaphore. + private sealed class GatedConnectionFactory : WaitHandleDbConnectionPoolTransactionTest.MockSqlConnectionFactory, IDisposable + { + internal readonly ManualResetEventSlim Entered = new(); + internal readonly ManualResetEventSlim Release = new(); + private int _calls; - Assert.True(t.Join(TimeSpan.FromSeconds(5)), "Waiter did not unblock within 5s of Shutdown."); - // Acceptable outcomes: either returned false/null (timed out / abandoned) or - // returned true/null (state-check short-circuit). Either way, it must NOT block - // forever, and it must NOT vend a real connection from a shut-down pool. - Assert.Null(waiterResult); - Assert.Null(waiterEx); - // Suppress unused warning - presence of waiterCompleted just documents the contract. - _ = waiterCompleted; + internal int CreateCount => Volatile.Read(ref _calls); + + protected override DbConnectionInternal CreateConnection(SqlConnectionOptions options, ConnectionPoolKey poolKey, + DbConnectionPoolGroupProviderInfo poolGroupProviderInfo, IDbConnectionPool pool, DbConnection owningConnection, TimeoutTimer timeout) + { + if (Interlocked.Increment(ref _calls) == 1) + { + Entered.Set(); + Assert.True(Release.Wait(TimeSpan.FromSeconds(15)), "Physical creation was not released."); + } + return base.CreateConnection(options, poolKey, poolGroupProviderInfo, pool, owningConnection, timeout); + } + + public void Dispose() + { + Entered.Dispose(); + Release.Dispose(); + } } // Startup() must be a no-op when the pool has already been shut down. Without the diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolTransactionTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolTransactionTest.cs index 5230f9e22e..2579343dda 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolTransactionTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolTransactionTest.cs @@ -651,7 +651,7 @@ public void PoolSaturation_BlocksUntilConnectionAvailable() // ----- Inner Stack Trace ----- // at Microsoft.Data.SqlClient.UnitTests.ConnectionPool.WaitHandleDbConnectionPoolTransactionTest.<>c__DisplayClass29_0.b__1() in WaitHandleDbConnectionPoolTransactionTest.cs:line 670 // at System.Threading.Tasks.Task.InnerInvoke() - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [Fact] public void TwoThreads_SharedTransaction_AccessSameTransactedEntry() { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Fixtures/AlwaysEncrypted/NativeColumnEncryptionKeyCertificateBaselineFixture.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Fixtures/AlwaysEncrypted/NativeColumnEncryptionKeyCertificateBaselineFixture.cs index d0cfc2a842..8994316343 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Fixtures/AlwaysEncrypted/NativeColumnEncryptionKeyCertificateBaselineFixture.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Fixtures/AlwaysEncrypted/NativeColumnEncryptionKeyCertificateBaselineFixture.cs @@ -51,15 +51,28 @@ public NativeColumnEncryptionKeyCertificateBaselineFixture() : base() { byte[] nativeCertificateBaseline = Resources.AlwaysEncrypted_NativeColumnEncryptionKeyBaseline_Certificate; + + // NOTE: The certificate is deliberately not disposed here. It is owned by the base fixture, which + // removes it from the store on cleanup - and a disposed certificate cannot be matched against a + // store, so disposing it early left it behind in the current user's store permanently. #if NET9_0_OR_GREATER - using X509Certificate2 certificate = X509CertificateLoader.LoadPkcs12(nativeCertificateBaseline, NativeCertificatePassword, + X509Certificate2 certificate = X509CertificateLoader.LoadPkcs12(nativeCertificateBaseline, NativeCertificatePassword, keyStorageFlags: X509KeyStorageFlags.PersistKeySet); #else - using X509Certificate2 certificate = new(nativeCertificateBaseline, NativeCertificatePassword, + X509Certificate2 certificate = new(nativeCertificateBaseline, NativeCertificatePassword, X509KeyStorageFlags.PersistKeySet); #endif - Thumbprint = certificate.Thumbprint; - AddToStore(certificate, StoreLocation.CurrentUser, StoreName.My); + try + { + Thumbprint = certificate.Thumbprint; + AddToStore(certificate, StoreLocation.CurrentUser, StoreName.My); + } + catch + { + Dispose(); + certificate.Dispose(); + throw; + } } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/ManagedSni/DataSourceNamedPipesTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/ManagedSni/DataSourceNamedPipesTests.cs new file mode 100644 index 0000000000..b06fe7e56d --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/ManagedSni/DataSourceNamedPipesTests.cs @@ -0,0 +1,212 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#if NET + +using Microsoft.Data.SqlClient.ManagedSni; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests.ManagedSni +{ + /// + /// Regression tests for Named Pipes data source parsing in . + /// + /// A UNC path host component may never contain a colon, so an IPv6 literal server name cannot + /// be used directly. Passing one through anyway composes a malformed pipe path such as + /// \\::1\pipe\sql\query, which sends the SMB redirector into an SMB session setup that + /// can fault LSASS on Windows and force a reboot. Windows instead defines a transcription for + /// this case (2001:db8::1 becomes 2001-db8--1.ipv6-literal.net), which the parser + /// now applies so IPv6 Named Pipes connections keep working. + /// + /// See: https://github.com/dotnet/SqlClient/issues/4523 + /// and https://learn.microsoft.com/openspecs/windows_protocols/ms-dtyp/62e862f4-2a51-452e-8eeb-dc4ff5ee33cc + /// + public class DataSourceNamedPipesTests + { + /// + /// Verifies that an IPv6 literal host is transcribed to its .ipv6-literal.net UNC + /// form, covering both the np:host form and the \\host\pipe\... UNC form, + /// with and without brackets and with a zone index. + /// + [Theory] + [InlineData(@"np:::1", "--1.ipv6-literal.net")] + [InlineData(@"np:[::1]", "--1.ipv6-literal.net")] + [InlineData(@"np:2001:db8::1", "2001-db8--1.ipv6-literal.net")] + [InlineData(@"np:fe80::1%3", "fe80--1s3.ipv6-literal.net")] + [InlineData(@"\\::1\pipe\sql\query", "--1.ipv6-literal.net")] + [InlineData(@"np:\\::1\pipe\sql\query", "--1.ipv6-literal.net")] + [InlineData(@"np:\\[2001:db8::1]\pipe\MSSQL$MYINSTANCE\sql\query", "2001-db8--1.ipv6-literal.net")] + public void ParseServerName_NamedPipesWithIPv6Literal_IsTranscribedToUncForm( + string dataSource, string expectedPipeHostName) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.Equal(DataSource.Protocol.NP, details.ResolvedProtocol); + Assert.Equal(expectedPipeHostName, details.PipeHostName); + // The pipe host name is what reaches the OS, so it must never carry a colon. + Assert.DoesNotContain(":", details.PipeHostName); + } + + /// + /// Verifies that a colon-bearing host that is not a parseable IPv6 literal has no UNC form + /// and is therefore rejected, rather than composing a malformed pipe path. + /// + [Theory] + [InlineData(@"np:not:a:host")] + [InlineData(@"np:2001:db8:::::1")] + [InlineData(@"\\not:a:host\pipe\sql\query")] + public void ParseServerName_NamedPipesWithUnparseableColonHost_IsRejected(string dataSource) + { + Assert.Null(DataSource.ParseServerName(dataSource)); + } + + /// + /// Verifies that IPv6 transcription does not regress legitimate Named Pipes data sources: + /// IPv4 literals, localhost, ., named instances, and explicit UNC pipe paths + /// must still parse and yield an unchanged pipe host name. + /// + [Theory] + [InlineData(@"np:127.0.0.1", "127.0.0.1")] + [InlineData(@"np:localhost", "localhost")] + [InlineData(@"np:.", ".")] + [InlineData(@"np:server\instance", "server")] + [InlineData(@"\\127.0.0.1\pipe\sql\query", "127.0.0.1")] + [InlineData(@"\\.\pipe\MSSQL$MYINSTANCE\sql\query", ".")] + [InlineData(@"\\my-server\pipe\sql\query", "my-server")] + public void ParseServerName_NamedPipesWithValidHost_IsAccepted( + string dataSource, string expectedPipeHostName) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.Equal(DataSource.Protocol.NP, details.ResolvedProtocol); + Assert.Equal(expectedPipeHostName, details.PipeHostName); + Assert.False(string.IsNullOrEmpty(details.PipeName)); + } + + /// + /// Verifies that a Named Pipes data source given without a UNC path still composes the + /// default pipe name, including the MSSQL$<instance> prefix for named instances. + /// These forms are asserted separately from the UNC forms because the UNC path builds its + /// pipe name with , which is platform dependent. + /// + [Theory] + [InlineData(@"np:127.0.0.1", @"sql\query")] + [InlineData(@"np:localhost", @"sql\query")] + [InlineData(@"np:::1", @"sql\query")] + [InlineData(@"np:server\instance", @"MSSQL$instance\sql\query")] + public void ParseServerName_NamedPipesWithoutUncPath_ComposesDefaultPipeName( + string dataSource, string expectedPipeName) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.Equal(expectedPipeName, details.PipeName); + } + + /// + /// Verifies that an IPv6 literal is preserved (unbracketed) in , + /// which feeds DNS resolution and SPN construction, while the pipe host name is transcribed. + /// The bracketed spelling must not survive into because + /// neither DNS nor SPN construction accepts it. + /// + [Theory] + [InlineData(@"np:2001:db8::1")] + [InlineData(@"np:[2001:db8::1]")] + [InlineData(@"np:\\2001:db8::1\pipe\sql\query")] + [InlineData(@"np:\\[2001:db8::1]\pipe\sql\query")] + public void ParseServerName_NamedPipesWithIPv6Literal_PreservesUnbracketedServerNameForSpn(string dataSource) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.Equal("2001:db8::1", details.ServerName); + Assert.Equal("2001-db8--1.ipv6-literal.net", details.PipeHostName); + } + + /// + /// Verifies unwraps bracketed IPv6 literals and + /// leaves every other host name untouched. + /// + [Theory] + [InlineData("[::1]", "::1")] + [InlineData("::1", "::1")] + [InlineData("[2001:db8::1]", "2001:db8::1")] + [InlineData("[fe80::1%3]", "fe80::1%3")] + [InlineData("localhost", "localhost")] + [InlineData("127.0.0.1", "127.0.0.1")] + [InlineData("not:a:host", "not:a:host")] + [InlineData("", "")] + public void NormalizeHostName_ReturnsExpected(string hostName, string expected) + { + Assert.Equal(expected, DataSource.NormalizeHostName(hostName)); + } + + /// + /// Without an explicit protocol prefix, managed SNI defaults to TCP, so an IPv6 literal + /// server name must continue to parse successfully and never reach the Named Pipes path. + /// + [Theory] + [InlineData("::1")] + [InlineData("[::1]")] + [InlineData("fe80::1")] + public void ParseServerName_IPv6LiteralWithoutProtocol_ResolvesToNonNamedPipes(string dataSource) + { + DataSource details = DataSource.ParseServerName(dataSource); + + Assert.NotNull(details); + Assert.NotEqual(DataSource.Protocol.NP, details.ResolvedProtocol); + Assert.Equal(dataSource, details.ServerName); + } + + /// + /// Verifies directly: colon-free host names + /// pass through untouched, IPv6 literals are transcribed per MS-DTYP, and colon-bearing host + /// names with no IPv6 interpretation return . + /// + [Theory] + [InlineData(".", ".")] + [InlineData("localhost", "localhost")] + [InlineData("127.0.0.1", "127.0.0.1")] + [InlineData("my-server.contoso.com", "my-server.contoso.com")] + [InlineData("--1.ipv6-literal.net", "--1.ipv6-literal.net")] + [InlineData("::1", "--1.ipv6-literal.net")] + [InlineData("[::1]", "--1.ipv6-literal.net")] + [InlineData("2001:db8::1", "2001-db8--1.ipv6-literal.net")] + [InlineData("::ffff:1.2.3.4", "--ffff-1.2.3.4.ipv6-literal.net")] + [InlineData("fe80::1%3", "fe80--1s3.ipv6-literal.net")] + public void GetUncCompatibleHostName_ReturnsExpected(string hostName, string expected) + { + Assert.Equal(expected, DataSource.GetUncCompatibleHostName(hostName)); + } + + /// + /// Verifies returns + /// for host names that contain a colon but have no IPv6 interpretation, and for empty input. + /// + [Theory] + [InlineData("not:a:host")] + [InlineData("2001:db8:::::1")] + [InlineData("[:]")] + [InlineData("")] + public void GetUncCompatibleHostName_UnconvertibleHost_ReturnsNull(string hostName) + { + Assert.Null(DataSource.GetUncCompatibleHostName(hostName)); + } + + /// + /// Verifies returns + /// for a null host name. Covered separately from the theory above because xUnit disallows + /// null theory data for a non-nullable string parameter. + /// + [Fact] + public void GetUncCompatibleHostName_Null_ReturnsNull() + { + Assert.Null(DataSource.GetUncCompatibleHostName(null)); + } + } +} + +#endif diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConfigurableRetryLogicLoaderTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConfigurableRetryLogicLoaderTest.cs new file mode 100644 index 0000000000..d4ef506991 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConfigurableRetryLogicLoaderTest.cs @@ -0,0 +1,388 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#if NET + +using System; +using System.IO; +using System.Reflection; +using System.Runtime.Loader; +using Microsoft.Data.SqlClient.Tests.Common; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests; + +/// +/// Unit tests validating that never leaves a +/// process-wide assembly resolving handler attached to +/// . +/// +/// +/// A handler left attached there participates in resolution of every assembly the host +/// application fails to find, not just the retry logic assembly the loader was interested in. +/// That silently changes assembly loading behaviour for code that never opted in to this +/// feature, and can serve unrelated assemblies out of this component's probing directory. +/// +[Collection(AppContextSwitchTestCollection.Name)] +public class SqlConfigurableRetryLogicLoaderTest +{ + /// + /// The default code path: no configuration at all. The loader must not subscribe to the + /// default load context. + /// + [Fact] + public void Constructor_WithNoConfiguration_DoesNotLeaveAssemblyProbingEnabled() + { + _ = new SqlConfigurableRetryLogicLoader(null, null); + + AssertNoAssemblyProbingHandlerInstalled(); + } + + /// + /// A configuration that supplies only a retry method, which is the documented way to select + /// one of the built-in retry providers. No custom type is being requested, so no assembly + /// resolution is required and no handler may be subscribed - not even transiently. + /// + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Constructor_WithoutRetryLogicType_DoesNotLeaveAssemblyProbingEnabled(string? retryLogicType) + { + TestRetryConnectionSection section = CreateSection(retryLogicType); + + SqlConfigurableRetryLogicLoader loader = new(section, null); + + // The built-in factory still resolves the requested method. + Assert.NotNull(loader.ConnectionProvider); + AssertNoAssemblyProbingHandlerInstalled(); + } + + /// + /// A configuration that requests a custom retry logic type legitimately needs assembly + /// resolution, but the handler must be removed again once type resolution has finished. + /// + [Fact] + public void Constructor_WithUnresolvableRetryLogicType_DoesNotLeaveAssemblyProbingEnabled() + { + TestRetryConnectionSection section = + CreateSection("Some.Namespace.NoSuchType, Some.Assembly.That.Does.Not.Exist"); + + SqlConfigurableRetryLogicLoader loader = new(section, null); + + // Resolution fails and falls back to the built-in factory rather than throwing. + Assert.NotNull(loader.ConnectionProvider); + AssertNoAssemblyProbingHandlerInstalled(); + } + + /// + /// The path a custom retry logic type is actually resolved on. This is the only path that + /// legitimately installs the probing handler, and it must be removed again once resolution + /// has finished. + /// + /// + /// The retry logic type is resolved out of this test assembly, which is reached through the + /// loader's probing directory rather than through normal assembly resolution. The invocation + /// counter confirms the configured type really was resolved and used, so this is exercising + /// the successful branch of type resolution rather than silently falling back to the built-in + /// factory. + /// + [Fact] + public void Constructor_WithResolvableRetryLogicType_DoesNotLeaveAssemblyProbingEnabled() + { + RunWithProbedRetryLogicFactory(loader => + { + Assert.NotNull(loader.ConnectionProvider); + AssertNoAssemblyProbingHandlerInstalled(); + }); + } + + /// + /// The probing handler must stay subscribed until the configured provider has been fully + /// constructed, not just until the type has been resolved. + /// + /// + /// Instantiating the configured type and invoking its retry method can trigger loads of that + /// assembly's private dependencies, and those loads happen after type resolution has already + /// returned. Unsubscribing too early would break providers that depend on that behaviour. The + /// factory method records the state of the handler at the moment it runs, which is inside the + /// window that has to remain open. + /// + [Fact] + public void RetryLogicTypeResolution_KeepsAssemblyProbingEnabledWhileProviderIsConstructed() + { + string probeAssemblyName = NewProbeAssemblyName(); + string plantedFile = PlantProbeFile(probeAssemblyName); + + ProbedRetryLogicFactory.ProbeAssemblyName = probeAssemblyName; + ProbedRetryLogicFactory.ProbingHandlerInstalledDuringInvocation = null; + + try + { + RunWithProbedRetryLogicFactory(loader => + { + Assert.NotNull(loader.ConnectionProvider); + + Assert.True( + ProbedRetryLogicFactory.ProbingHandlerInstalledDuringInvocation, + "The assembly probing handler was not subscribed while the configured retry " + + "logic provider was being constructed."); + }); + + // ...and it must be gone again afterwards. + AssertNoAssemblyProbingHandlerInstalled(); + } + finally + { + ProbedRetryLogicFactory.ProbeAssemblyName = null; + ProbedRetryLogicFactory.ProbingHandlerInstalledDuringInvocation = null; + DeleteProbeFile(plantedFile); + } + } + + /// + /// Disposing an + /// removes its handler from the default assembly load context. + /// + [Fact] + public void AssemblyResolutionSubscription_DisposeRemovesAssemblyProbingHandler() + { + string subscribedProbeName = NewProbeAssemblyName(); + string subscribedProbeFile = PlantProbeFile(subscribedProbeName); + string disposedProbeName = NewProbeAssemblyName(); + string disposedProbeFile = PlantProbeFile(disposedProbeName); + + try + { + using SqlConfigurableRetryLogicLoader.AssemblyResolutionSubscription subscription = + new(subscribe: true); + + Assert.True(IsProbingHandlerInstalled(subscribedProbeName)); + + subscription.Dispose(); + + Assert.False(IsProbingHandlerInstalled(disposedProbeName)); + } + finally + { + DeleteProbeFile(subscribedProbeFile); + DeleteProbeFile(disposedProbeFile); + } + } + + /// + /// Builds a configuration that resolves out of this test + /// assembly through the loader's probing directory, constructs a loader from it, and hands the + /// loader to . + /// + private static void RunWithProbedRetryLogicFactory(Action assert) + { + Assembly testAssembly = typeof(SqlConfigurableRetryLogicLoaderTest).Assembly; + string assemblySimpleName = testAssembly.GetName().Name!; + + // The loader probes for '.dll'. The test assembly's file name does not + // necessarily match its simple name, so make a copy that does. + string probePath = Path.Combine(AppContext.BaseDirectory, assemblySimpleName + ".dll"); + bool copied = false; + if (!File.Exists(probePath)) + { + File.Copy(testAssembly.Location, probePath); + copied = true; + } + + try + { + TestRetryConnectionSection section = CreateSection( + $"{typeof(ProbedRetryLogicFactory).FullName}, {assemblySimpleName}"); + section.RetryMethod = nameof(ProbedRetryLogicFactory.CreateProbedRetryProvider); + + ProbedRetryLogicFactory.InvocationCount = 0; + + SqlConfigurableRetryLogicLoader loader = new(section, null); + + Assert.Equal(1, ProbedRetryLogicFactory.InvocationCount); + + assert(loader); + } + finally + { + if (copied) + { + DeleteProbeFile(probePath); + } + } + } + + private static TestRetryConnectionSection CreateSection(string? retryLogicType) => + new() + { + RetryLogicType = retryLogicType!, + RetryMethod = nameof(SqlConfigurableRetryFactory.CreateFixedRetryProvider), + NumberOfTries = 2, + DeltaTime = TimeSpan.FromSeconds(1), + MinTimeInterval = TimeSpan.Zero, + MaxTimeInterval = TimeSpan.FromSeconds(10), + }; + + /// + /// Asserts that a failed assembly load is not served out of the loader's probing directory, + /// which can only happen while a resolving handler installed by + /// is subscribed to + /// . + /// + /// + /// A file that is not a valid assembly is planted in the probing directory under a name no + /// other component could be asking for. If a handler is still subscribed it finds that file + /// and tries to load it, which surfaces as . With no + /// handler subscribed the runtime never looks there and reports the assembly as simply not + /// found. This asserts the behaviour that actually matters to a host application rather than + /// inspecting loader or runtime internals. + /// + private static void AssertNoAssemblyProbingHandlerInstalled() + { + string assemblySimpleName = NewProbeAssemblyName(); + string plantedFile = PlantProbeFile(assemblySimpleName); + + try + { + Assert.False( + IsProbingHandlerInstalled(assemblySimpleName), + "A resolving handler that probes the loader's probing directory is subscribed to " + + "the default assembly load context."); + } + finally + { + DeleteProbeFile(plantedFile); + } + } + + /// + /// Reports whether a resolving handler that serves assemblies out of the loader's probing + /// directory is currently subscribed to . + /// + /// + /// This distinguishes the two states using only public behaviour, which is what actually + /// matters to a host application. With such a handler subscribed the planted file is found + /// and an attempt is made to load it, which fails as + /// because it is not a valid assembly. With no such + /// handler subscribed the runtime never looks in that directory and reports the assembly as + /// simply not found. + /// + internal static bool IsProbingHandlerInstalled(string assemblySimpleName) + { + try + { + Assembly.Load(new AssemblyName(assemblySimpleName)); + + // Unreachable: the planted file is deliberately not a valid assembly, so a handler + // that found it cannot have loaded it successfully. + return true; + } + catch (BadImageFormatException) + { + return true; + } + catch (FileNotFoundException) + { + return false; + } + } + + private static string NewProbeAssemblyName() => + "MdsProbeAssembly_" + Guid.NewGuid().ToString("N"); + + /// + /// Plants a file that is not a valid assembly in the loader's probing directory, under a + /// name no other component could be asking for. + /// + private static string PlantProbeFile(string assemblySimpleName) + { + // The probing directory is the application base directory, which for a test run is the + // directory the test assembly was loaded from. + string plantedFile = Path.Combine(AppContext.BaseDirectory, assemblySimpleName + ".dll"); + + File.WriteAllText(plantedFile, "not an assembly"); + + return plantedFile; + } + + /// + /// Removes a planted probe file. Cleanup failures are ignored so that a test reports on + /// product behaviour rather than on the state of the file system. + /// + private static void DeleteProbeFile(string plantedFile) + { + try + { + File.Delete(plantedFile); + } + catch (IOException) + { + // The file is in use, most likely because it was successfully loaded as an assembly + // and is therefore locked for the lifetime of the process. + } + catch (UnauthorizedAccessException) + { + // The file is read only, or the caller lacks permission to delete it. + } + } + + private sealed class TestRetryConnectionSection : ISqlConfigurableRetryConnectionSection + { + public TimeSpan DeltaTime { get; set; } + + public TimeSpan MaxTimeInterval { get; set; } + + public TimeSpan MinTimeInterval { get; set; } + + public int NumberOfTries { get; set; } + + public string RetryLogicType { get; set; } = string.Empty; + + public string RetryMethod { get; set; } = string.Empty; + + public string TransientErrors { get; set; } = string.Empty; + } +} + +/// +/// A retry logic factory that is resolved through the loader's probing directory rather than +/// through normal assembly resolution, so tests can tell a successful custom type resolution +/// apart from a silent fallback to the built-in factory. +/// +/// +/// This has to be a public, non-nested type because the loader discovers candidates by +/// enumerating the resolved assembly's exported types. +/// +public static class ProbedRetryLogicFactory +{ + internal static int InvocationCount; + + /// + /// When set to the simple name of a planted probe assembly, the factory method records whether + /// the loader's probing handler was subscribed at the moment it ran. + /// + internal static string? ProbeAssemblyName; + + /// + /// The state of the loader's probing handler at the moment the factory method last ran, or null + /// if it was not recorded. + /// + internal static bool? ProbingHandlerInstalledDuringInvocation; + + public static SqlRetryLogicBaseProvider CreateProbedRetryProvider(SqlRetryLogicOption option) + { + InvocationCount++; + + if (ProbeAssemblyName is not null) + { + ProbingHandlerInstalledDuringInvocation = + SqlConfigurableRetryLogicLoaderTest.IsProbingHandlerInstalled(ProbeAssemblyName); + } + + return SqlConfigurableRetryFactory.CreateFixedRetryProvider(option); + } +} + +#endif diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionConcurrentOpenTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionConcurrentOpenTests.cs index 17ac619a3e..7515878235 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionConcurrentOpenTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionConcurrentOpenTests.cs @@ -61,6 +61,25 @@ public void InnerConnection_InConnectingState_ReportsConnectingState() Assert.Equal(ConnectionState.Connecting, connection.State); } + /// + /// Verifies the registered application cannot change while a connection is opening and + /// that a rejected assignment preserves the original value. + /// + [Fact] + public void RegisteredApplication_WhenConnecting_ThrowsAndPreservesValue() + { + using var connection = new SqlConnection("Data Source=localhost") + { + RegisteredApplication = RegisteredApplication.EntityFrameworkCore, + }; + ForceInnerConnection(connection, GetConnectingSingleton()); + + Assert.Throws( + () => connection.RegisteredApplication = RegisteredApplication.SemanticKernel); + + Assert.Equal(RegisteredApplication.EntityFrameworkCore, connection.RegisteredApplication); + } + [Fact] public void Open_WhenAlreadyConnecting_ThrowsInvalidOperation() { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionInternalResetTransactionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionInternalResetTransactionTests.cs new file mode 100644 index 0000000000..dcf7262e78 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionInternalResetTransactionTests.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Data.SqlClient.Connection; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests; + +/// +/// Verifies , which decides +/// whether a pooled connection is reset with ST_RESET_CONNECTION_PRESERVE_TRANSACTION +/// rather than a plain ST_RESET_CONNECTION. +/// +/// Getting this predicate wrong has caused two separate regressions, in opposite directions: +/// +/// +/// Testing only IsTransactionRoot misses connections that merely enlisted in someone +/// else's transaction, so they are reset and subsequently run in auto-commit mode +/// (https://github.com/dotnet/SqlClient/issues/2970). +/// +/// +/// Testing only EnlistedTransaction misses delegated transaction roots, whose +/// server-side transaction is then reset out from under System.Transactions, corrupting the +/// connection as it is recycled through the pool +/// (https://github.com/dotnet/SqlClient/issues/4001). +/// +/// +/// +/// Both conditions must therefore be honored. Asserting against the extracted predicate covers +/// every combination deterministically, including state combinations that are transient and +/// racy to stage against a live server. +/// +public class SqlConnectionInternalResetTransactionTests +{ + /// + /// Exhaustively pins the predicate over all eight combinations of its three boolean inputs. + /// + /// The expectations encode two rules: + /// + /// An unpooled connection is never recycled, so nothing is ever preserved. + /// A pooled connection tied to a transaction in either way must be preserved. + /// + /// + [Theory] + // An unpooled connection is destroyed rather than recycled, so there is no subsequent use to + // protect and its transaction must not be preserved, regardless of any other state. + [InlineData(false, false, false, false)] + [InlineData(false, false, true, false)] + [InlineData(false, true, false, false)] + [InlineData(false, true, true, false)] + // Pooled, no transaction of any kind: nothing to preserve. + [InlineData(true, false, false, false)] + // Regression guard for #2970: an implementation keyed only on IsTransactionRoot returns false + // for a pooled connection enlisted in a transaction it does not own. + [InlineData(true, false, true, true)] + // Regression guard for #4001: an implementation keyed only on EnlistedTransaction returns + // false in this transient half-state, after the enlistment is detached but while the delegated + // transaction still reports itself as active. + [InlineData(true, true, false, true)] + // Pooled connection that is both a delegated root and has an EnlistedTransaction. This is + // the common state immediately after enlistment, since enlistment sets EnlistedTransaction + // unconditionally. + [InlineData(true, true, true, true)] + public void ShouldPreserveTransactionOnReset_CoversBothTransactionOwnershipModes( + bool isPooled, + bool isTransactionRoot, + bool hasEnlistedTransaction, + bool expected) + { + // Act + bool actual = SqlConnectionInternal.ShouldPreserveTransactionOnReset( + isPooled: isPooled, + isTransactionRoot: isTransactionRoot, + hasEnlistedTransaction: hasEnlistedTransaction); + + // Assert + Assert.Equal(expected, actual); + } + +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs index 11b4ed8637..4c15a65789 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlConnectionOptionsTest.cs @@ -59,7 +59,9 @@ public void TestDefaultTnir(string dataSource, bool? tnirEnabledInConnString, bo builder.DataSource = dataSource; if (tnirEnabledInConnString.HasValue) { +#pragma warning disable 618 // TransparentNetworkIPResolution is obsolete builder.TransparentNetworkIPResolution = tnirEnabledInConnString.Value; +#pragma warning restore 618 } SqlConnectionOptions connectionString = new(builder.ConnectionString); @@ -99,7 +101,9 @@ public void TestShouldDisableTnirWithCallerSuppliedToken( SqlConnectionStringBuilder builder = new() { DataSource = dataSource }; if (tnirInConnString.HasValue) { +#pragma warning disable 618 // TransparentNetworkIPResolution is obsolete builder.TransparentNetworkIPResolution = tnirInConnString.Value; +#pragma warning restore 618 } SqlConnectionOptions connectionOptions = new(builder.ConnectionString); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataFactoryDataTypesTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataFactoryDataTypesTest.cs new file mode 100644 index 0000000000..4a0742e17d --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlMetaDataFactoryDataTypesTest.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Data; +using System.Data.Common; +using System.IO; +using System.Linq; +using System.Reflection; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests; + +/// +/// Tests that correctly reports the json data type +/// in the DataTypes schema collection based on . +/// +public sealed class SqlMetaDataFactoryDataTypesTest +{ + private static Stream GetMetaDataXmlStream() + { + Assembly assembly = typeof(SqlConnection).Assembly; + Stream? stream = assembly.GetManifestResourceStream("Microsoft.Data.SqlClient.SqlMetaData.xml"); + Assert.NotNull(stream); + return stream; + } + + private static SqlMetaDataFactory CreateFactory(bool jsonTypeSupported) + { + Stream stream = GetMetaDataXmlStream(); + ConnectionCapabilities capabilities = new() + { + TdsVersion = TdsEnums.TDS7X_VERSION, + ServerMajorVersion = 12, + ServerMinorVersion = 0, + ServerBuildNumber = 0, + JsonType = jsonTypeSupported + }; + return new SqlMetaDataFactory(stream, capabilities); + } + + private static bool DataTypesContainsJson(SqlMetaDataFactory factory) + { + FieldInfo? field = typeof(SqlMetaDataFactory) + .GetField("_collectionDataSet", BindingFlags.NonPublic | BindingFlags.Instance); + Assert.NotNull(field); + + DataSet? dataSet = (DataSet?)field.GetValue(factory); + Assert.NotNull(dataSet); + + DataTable? dataTypes = dataSet.Tables[DbMetaDataCollectionNames.DataTypes]; + Assert.NotNull(dataTypes); + + DataColumn? typeNameColumn = dataTypes.Columns[DbMetaDataColumnNames.TypeName]; + Assert.NotNull(typeNameColumn); + + return dataTypes.Rows + .Cast() + .Any(row => (string)row[typeNameColumn] == "json"); + } + + /// + /// Verifies that json appears in the DataTypes schema collection when + /// is . + /// + [Fact] + public void DataTypesTable_ContainsJson_WhenJsonTypeSupported() + { + using SqlMetaDataFactory factory = CreateFactory(jsonTypeSupported: true); + + Assert.True(DataTypesContainsJson(factory)); + } + + /// + /// Verifies that json does not appear in the DataTypes schema collection when + /// is , covering + /// Azure SQL which always reports version 12.x regardless of json support. + /// + [Fact] + public void DataTypesTable_DoesNotContainJson_WhenJsonTypeNotSupported() + { + using SqlMetaDataFactory factory = CreateFactory(jsonTypeSupported: false); + + Assert.False(DataTypesContainsJson(factory)); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/RegisteredApplicationTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/RegisteredApplicationTests.cs new file mode 100644 index 0000000000..e036ae84bd --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/RegisteredApplicationTests.cs @@ -0,0 +1,161 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.Data.SqlClient; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests; + +/// +/// Tests for the application identifier registry. +/// +public class RegisteredApplicationTests +{ + /// + /// Verifies the enum uses the 16-bit protocol width. + /// + [Fact] + public void UnderlyingType_Is_UShort() + { + Assert.Equal(typeof(ushort), Enum.GetUnderlyingType(typeof(RegisteredApplication))); + } + + /// + /// Verifies the default value reports no application identity. + /// + [Fact] + public void Default_Is_Unknown() + { + Assert.Equal(RegisteredApplication.Unknown, default(RegisteredApplication)); + Assert.Equal(0, (int)RegisteredApplication.Unknown); + } + + /// + /// Verifies the reserved identifiers keep their assigned values, since + /// changing one would silently re-map an application's telemetry. + /// + [Theory] + [InlineData(RegisteredApplication.EntityFrameworkCore, 1)] + [InlineData(RegisteredApplication.SemanticKernel, 2)] + [InlineData(RegisteredApplication.ManagementStudio, 3)] + [InlineData(RegisteredApplication.SqlManagementObjects, 4)] + [InlineData(RegisteredApplication.DataTierApplicationFramework, 5)] + [InlineData(RegisteredApplication.SqlToolsService, 6)] + [InlineData(RegisteredApplication.AspNetCoreDistributedSqlServerCache, 7)] + [InlineData(RegisteredApplication.EntityFramework, 8)] + [InlineData(RegisteredApplication.AzureFunctionsSqlExtension, 9)] + [InlineData(RegisteredApplication.OrleansAdoNet, 10)] + [InlineData(RegisteredApplication.DurableTaskSqlServer, 11)] + [InlineData(RegisteredApplication.SqlPackage, 12)] + [InlineData(RegisteredApplication.DataApiBuilder, 13)] + public void Members_Have_Stable_Values(RegisteredApplication app, int expected) + { + Assert.Equal(expected, (int)app); + } + + /// + /// Verifies an unregistered identifier can be reported by casting, which + /// keeps the API forward compatible with identifiers added later. + /// + [Fact] + public void Unregistered_Identifier_Is_Accepted() + { + RegisteredApplication app = (RegisteredApplication)0xC001; + + Assert.False(Enum.IsDefined(typeof(RegisteredApplication), app)); + + using SqlConnection connection = new(); + connection.RegisteredApplication = app; + + Assert.Equal(app, connection.RegisteredApplication); + } + + /// + /// Verifies the boundaries of the 16-bit identifier space are accepted, + /// since the payload reports the identifier in exactly 16 bits. + /// + [Theory] + [InlineData(0)] + [InlineData(ushort.MaxValue)] + public void Identifier_In_Range_Is_Accepted(int value) + { + using SqlConnection connection = new(); + + connection.RegisteredApplication = (RegisteredApplication)value; + + Assert.Equal(value, (int)connection.RegisteredApplication); + } + + /// + /// Verifies the connection reports no application identity until one is + /// assigned, and round-trips the value it is given. + /// + [Fact] + public void SqlConnection_RegisteredApplication_RoundTrips() + { + using SqlConnection connection = new(); + + Assert.Equal(RegisteredApplication.Unknown, connection.RegisteredApplication); + + connection.RegisteredApplication = RegisteredApplication.SemanticKernel; + + Assert.Equal(RegisteredApplication.SemanticKernel, connection.RegisteredApplication); + } + + /// + /// Verifies a cloned connection keeps the application identity of the + /// connection it was cloned from, so cloning does not silently drop the + /// identity back to . + /// + [Fact] + public void Clone_Preserves_RegisteredApplication() + { + using SqlConnection connection = new(); + connection.RegisteredApplication = RegisteredApplication.SqlPackage; + + using SqlConnection clone = (SqlConnection)((ICloneable)connection).Clone(); + + Assert.Equal(RegisteredApplication.SqlPackage, clone.RegisteredApplication); + } + + /// + /// Verifies the driver properties part reports the connection pool V2 flag + /// when, and only when, that implementation is enabled. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public void DriverProperties_Reports_ConnectionPoolV2(bool useConnectionPoolV2) + { + SqlClientDriverProperties expected = useConnectionPoolV2 + ? SqlClientDriverProperties.ConnectionPoolV2 + : SqlClientDriverProperties.None; + + Assert.Equal(expected, SqlClientDriverPropertiesResolver.Resolve(useConnectionPoolV2)); + } + + /// + /// Verifies driver properties reserve 64 bits for feature flags. + /// + [Fact] + public void DriverProperties_UnderlyingType_Is_ULong() + { + Assert.Equal(typeof(ulong), Enum.GetUnderlyingType(typeof(SqlClientDriverProperties))); + } + + /// + /// Verifies the flags reported for this process agree with the switch they + /// are derived from, so + /// cannot drift from the mapping it delegates to. + /// + [Fact] + public void DriverProperties_Current_Matches_Switch() + { + SqlClientDriverProperties expected = + SqlClientDriverPropertiesResolver.Resolve(LocalAppContextSwitches.UseConnectionPoolV2); + + Assert.Equal(expected, SqlClientDriverPropertiesResolver.Current); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs index ba1a852626..7eeb5df3d8 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs @@ -37,7 +37,7 @@ public class ConnectionFailoverTests // at Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests.ConnectionFailoverTests.TransientFault_NoFailover_DoesNotClearPool(UInt32 errorCode) in D:\a\_work\1\s\src\Microsoft.Data.SqlClient\tests\UnitTests\SimulatedServerTests\ConnectionFailoverTests.cs:line 72 // at System.RuntimeMethodHandle.InvokeMethod(Object target, Void** arguments, Signature sig, Boolean isConstructor) // at System.Reflection.MethodBaseInvoker.InvokeDirectByRefWithFewArgs(Object obj, Span`1 copyOfArgs, BindingFlags invokeAttr) - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [Theory] [InlineData(40613)] [InlineData(42108)] @@ -129,7 +129,9 @@ public void NetworkError_TriggersFailover_ClearsPool() InitialCatalog = "test", MultiSubnetFailover = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; @@ -199,7 +201,9 @@ public void NetworkTimeout_ShouldFail() Encrypt = false, MultiSubnetFailover = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); @@ -246,7 +250,9 @@ public void NetworkDelay_ShouldConnectToPrimary() Pooling = false, // Disable pooling to ensure a fresh connection attempt is made MultiSubnetFailover = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); @@ -299,7 +305,7 @@ public void NetworkDelay_ShouldConnectToPrimary() // at Microsoft.Data.SqlClient.SqlConnection.Open(SqlConnectionOverrides overrides) // at Microsoft.Data.SqlClient.SqlConnection.Open() // at Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests.ConnectionFailoverTests.NetworkError_WithUserProvidedPartner_RetryDisabled_ShouldConnectToFailoverPartner() in ConnectionFailoverTests.cs:line 304 - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [Fact] public void NetworkError_WithUserProvidedPartner_RetryDisabled_ShouldConnectToFailoverPartner() { @@ -388,7 +394,9 @@ public void NetworkError_WithUserProvidedPartner_RetryEnabled_ShouldConnectToFai FailoverPartner = $"localhost,{failoverServer.EndPoint.Port}", // User provided failover partner Encrypt = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); @@ -805,7 +813,7 @@ public async Task TransientFault_WithUserProvidedPartner_Async_ShouldConnectToPr // at Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests.ConnectionFailoverTests.TransientFault_WithUserProvidedPartner_Pooling_ShouldNotClearPool_NotFailover(UInt32 errorCode) in D:\a\_work\1\s\src\Microsoft.Data.SqlClient\tests\UnitTests\SimulatedServerTests\ConnectionFailoverTests.cs:line 777 // at InvokeStub_ConnectionFailoverTests.TransientFault_WithUserProvidedPartner_Pooling_ShouldNotClearPool_NotFailover(Object, Span`1) // at System.Reflection.MethodBaseInvoker.InvokeWithOneArg(Object obj, BindingFlags invokeAttr, Binder binder, Object[] parameters, CultureInfo culture) - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [Theory] [InlineData(40613)] [InlineData(42108)] diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs index 262770f563..918f7228f3 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs @@ -136,7 +136,9 @@ public void NetworkDelayAtRoutedLocation_RetryDisabled_ShouldSucceed(bool multiS Encrypt = false, MultiSubnetFailover = multiSubnetFailoverEnabled, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = multiSubnetFailoverEnabled, + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); @@ -190,7 +192,9 @@ public void NetworkTimeoutAtRoutedLocation_RetryDisabled_ShouldFail() Encrypt = false, MultiSubnetFailover = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false + #pragma warning restore 618 #endif }; using SqlConnection connection = new(builder.ConnectionString); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs index f797797758..a8216d787f 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionTests.cs @@ -25,6 +25,7 @@ namespace Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests { + [Collection(SimulatedServerTestCollection.Name)] public class ConnectionTests { [Fact] @@ -92,7 +93,7 @@ public async Task RequestEncryption_ServerDoesNotSupportEncryption_ShouldFail() // at Microsoft.Data.SqlClient.Connection.SqlConnectionInternal.OnError(...) // at Microsoft.Data.SqlClient.Connection.SqlConnectionInternal.CompleteLogin(Boolean enlistOK) // at Microsoft.Data.SqlClient.Connection.SqlConnectionInternal.LoginNoFailover(...) - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [Theory] [InlineData(40613)] [InlineData(42108)] @@ -111,7 +112,9 @@ public async Task TransientFault_RetryEnabled_ShouldSucceed_Async(uint errorCode DataSource = "localhost," + server.EndPoint.Port, Encrypt = SqlConnectionEncryptOption.Optional, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false + #pragma warning restore 618 #endif }; @@ -231,7 +234,7 @@ public void TransientFault_RetryDisabled_ShouldFail(uint errorCode) // at Microsoft.Data.SqlClient.SqlConnectionFactory.CreateNonPooledConnection(DbConnection owningConnection, DbConnectionPoolGroup poolGroup, TimeoutTimer timeout) // at Microsoft.Data.SqlClient.SqlConnectionFactory.<>c__DisplayClass41_0.b__0(Task`1 _) // at Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests.ConnectionTests.NetworkError_RetryEnabled_ShouldSucceed_Async(Boolean multiSubnetFailoverEnabled) - [Trait("Category", "flaky")] + [Trait("category", "flaky")] [Theory] [InlineData(false)] [InlineData(true)] @@ -252,7 +255,9 @@ public async Task NetworkError_RetryEnabled_ShouldSucceed_Async(bool multiSubnet Pooling = false, // Disable pooling to ensure a fresh connection attempt is made MultiSubnetFailover = multiSubnetFailoverEnabled, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = multiSubnetFailoverEnabled + #pragma warning restore 618 #endif }; @@ -290,7 +295,9 @@ public async Task NetworkDelay_RetryDisabled_Async(bool multiSubnetFailoverEnabl Encrypt = SqlConnectionEncryptOption.Optional, MultiSubnetFailover = multiSubnetFailoverEnabled, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = multiSubnetFailoverEnabled, + #pragma warning restore 618 #endif }; @@ -338,7 +345,9 @@ public void NetworkDelay_RetryDisabled(bool multiSubnetFailoverEnabled) ConnectTimeout = 5, MultiSubnetFailover = multiSubnetFailoverEnabled, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = multiSubnetFailoverEnabled, + #pragma warning restore 618 #endif }; @@ -894,33 +903,41 @@ public void SspiContextProviderAndAccessTokenStateAreMutuallyExclusive() { Func> callback = (ctx, token) => Task.FromResult(new SqlAuthenticationToken("invalid", DateTimeOffset.MaxValue)); + string expectedMessage = global::Microsoft.Data.StringsHelper.GetString( + global::System.Strings.ADP_InvalidMixedUsageOfAccessTokenProperties); // Token first, then provider. using (SqlConnection conn = new("Data Source=localhost")) { conn.AccessToken = "token"; - Assert.Throws( + InvalidOperationException exception = Assert.Throws( () => conn.SspiContextProvider = new TestSspiContextProvider()); + Assert.Equal(expectedMessage, exception.Message); } using (SqlConnection conn = new("Data Source=localhost")) { conn.AccessTokenCallback = callback; - Assert.Throws( + InvalidOperationException exception = Assert.Throws( () => conn.SspiContextProvider = new TestSspiContextProvider()); + Assert.Equal(expectedMessage, exception.Message); } // Provider first, then token. using (SqlConnection conn = new("Data Source=localhost")) { conn.SspiContextProvider = new TestSspiContextProvider(); - Assert.Throws(() => conn.AccessToken = "token"); + InvalidOperationException exception = Assert.Throws( + () => conn.AccessToken = "token"); + Assert.Equal(expectedMessage, exception.Message); } using (SqlConnection conn = new("Data Source=localhost")) { conn.SspiContextProvider = new TestSspiContextProvider(); - Assert.Throws(() => conn.AccessTokenCallback = callback); + InvalidOperationException exception = Assert.Throws( + () => conn.AccessTokenCallback = callback); + Assert.Equal(expectedMessage, exception.Message); } } @@ -1162,15 +1179,20 @@ public void TestConnWithVectorFeatExtVersionNegotiation(bool expectedConnectionR } } - // Test that the driver sends the UserAgent feature extension when - // the context switch is enabled, and that the presence or absence of - // an ack from the server has no effect. + /// + /// Verifies that LOGIN7 sends the USERAGENT payload carrying the connection's application + /// identity, regardless of whether the server acknowledges the extension. + /// + /// Whether the server acknowledges the USERAGENT extension. + /// Whether the connection opens asynchronously. [Theory] // Allow the server to ack. - [InlineData(true)] + [InlineData(true, false)] + [InlineData(true, true)] // Don't allow the server to send an ack. - [InlineData(false)] - public void TestConnWithUserAgentFeatureExtension(bool sendAck) + [InlineData(false, false)] + [InlineData(false, true)] + public async Task TestConnWithUserAgentFeatureExtension(bool sendAck, bool useAsync) { // Start the test server. using TdsServer server = new(); @@ -1219,7 +1241,15 @@ public void TestConnWithUserAgentFeatureExtension(bool sendAck) }.ConnectionString; using var connection = new SqlConnection(connStr); - connection.Open(); + connection.RegisteredApplication = RegisteredApplication.EntityFrameworkCore; + if (useAsync) + { + await connection.OpenAsync(); + } + else + { + connection.Open(); + } // Verify the connection itself succeeded Assert.Equal(ConnectionState.Open, connection.State); @@ -1229,9 +1259,38 @@ public void TestConnWithUserAgentFeatureExtension(bool sendAck) Assert.True(firstFeatureIsUserAgent); Assert.True(tokenWasNotNull); Assert.True(dataLengthAtLeast1); - Assert.Equal(UserAgent.Ucs2Bytes.ToArray(), observedPayload); + Assert.Equal(UserAgent.GetUcs2Bytes(RegisteredApplication.EntityFrameworkCore).ToArray(), observedPayload); // TODO: Confirm the server sent an Ack by reading log message from SqlInternalConnectionTds } + + /// + /// Verifies the application identity cannot be changed once the connection is open, since + /// it is only reported during login and the getter would otherwise report a value that was + /// never sent. + /// + [Fact] + public void RegisteredApplication_CannotBeSet_WhenConnectionIsOpen() + { + using TdsServer server = new(); + server.Start(); + + var connStr = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + Pooling = false, + }.ConnectionString; + + using var connection = new SqlConnection(connStr); + connection.RegisteredApplication = RegisteredApplication.EntityFrameworkCore; + connection.Open(); + + Assert.Throws( + () => connection.RegisteredApplication = RegisteredApplication.SemanticKernel); + + // The connection still reports the identity it logged in with. + Assert.Equal(RegisteredApplication.EntityFrameworkCore, connection.RegisteredApplication); + } } } diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/PoolClearDuringOpenTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/PoolClearDuringOpenTests.cs new file mode 100644 index 0000000000..9b6bf1f2b4 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/PoolClearDuringOpenTests.cs @@ -0,0 +1,205 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient.ConnectionPool; +using Microsoft.Data.SqlClient.Tests.Common; +using Microsoft.SqlServer.TDS.Servers; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests; + +/// +/// Exercises public pool clearing while one login is in flight and another open is waiting. +/// +[Collection(SimulatedServerTestCollection.Name)] +public class PoolClearDuringOpenTests +{ + /// + /// Blocks the first login, admits a second open, then clears the pool before releasing + /// the login. Both opens finish on the retired pool and their connections are destroyed + /// on close. A subsequent open uses a replacement pool. + /// + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ClearDuringOpen_InFlightConnectionsAreDestroyedOnClose(bool async, bool clearAll) + { + // Arrange + // Initiate an open using the wait-handle pool. Its login will be blocked by the gated server. + using var switches = new LocalAppContextSwitchesHelper { UseConnectionPoolV2 = false }; + using var server = new GatedLoginServer(); + using var first = new SqlConnection(server.ConnectionString); + using var pending = new SqlConnection(server.ConnectionString); + using var replacement = new SqlConnection(server.ConnectionString); + Task opens = Open(first, async); + + try + { + // Make sure the first open is blocked during login. Then, initiate a second open. + // The second open must wait for the first creation to finish. + server.WaitForFirstLogin(); + var retiredPool = Assert.IsType(first.PoolGroup.GetConnectionPool(SqlConnectionFactory.Instance)); + Assert.False(opens.IsCompleted); + Task pendingOpen = Open(pending, async); + opens = Task.WhenAll(opens, pendingOpen); + + // Make sure the second open is pending on the same pool and has not started its login. + AssertPendingOpen(pending, pendingOpen, retiredPool, async); + Assert.Same(first.PoolGroup, pending.PoolGroup); + Assert.Equal(1, server.LoginCount); + + // Act + // Now, clear the pool. New opens should use a new pool, but these in-flight opens should proceed. + if (clearAll) + { + SqlConnection.ClearAllPools(); + } + else + { + SqlConnection.ClearPool(first); + } + + // Unblock the first login, allowing both opens to proceed, in turn. + server.ReleaseFirstLogin(); + + // Assert + // Wait for both opens to complete, then confirm they succeeded on the original pool. + await AssertCompletes(opens); + await opens; + + Assert.False(retiredPool.IsRunning); + Assert.Equal(ConnectionState.Open, first.State); + Assert.Equal(ConnectionState.Open, pending.State); + Assert.Same(retiredPool, first.InnerConnection.Pool); + Assert.Same(retiredPool, pending.InnerConnection.Pool); + + // Assert that both opens created new connections. + Assert.Equal(2, retiredPool.Count); + Assert.Equal(2, server.LoginCount); + + // Act + // Initiate another open after clearing. This one should use a new pool. + await Open(replacement, async); + + // Assert + // Make sure the new open used a different, running pool and performed another login. + Assert.NotSame(retiredPool, replacement.InnerConnection.Pool); + Assert.True(replacement.InnerConnection.Pool.IsRunning); + Assert.Equal(3, server.LoginCount); + + // Act + // Close the connections that completed on the retired pool. They should be destroyed, not reused. + first.Close(); + pending.Close(); + + // Assert: the retired pool has no connections or pending requests left. + Assert.Equal(0, retiredPool.Count); + Assert.Equal(0, retiredPool.IdleCount); + Assert.Equal(0, Volatile.Read(ref retiredPool._waitCount)); + } + finally + { + // Cleanup + server.ReleaseFirstLogin(); + await AssertCompletes(opens); + if (opens.IsFaulted) + { + // Observe background failures without replacing an earlier assertion failure. + _ = opens.Exception; + } + first.Close(); + pending.Close(); + replacement.Close(); + SqlConnection.ClearPool(first); + } + } + + /// Confirms the second open is admitted before the test clears its pool. + private static void AssertPendingOpen(SqlConnection pending, Task open, WaitHandleDbConnectionPool pool, bool async) + { + if (async) + { + // The second async open is queued behind the worker handling the first login. + // It has not entered the pool's wait loop yet, so _waitCount will still be 1. + Assert.Equal(ConnectionState.Connecting, pending.State); + } + else + { + // The sync opens run on separate threads. Wait until the second has entered + // the pool and is blocked on the creation semaphore before allowing the test to clear it. + Assert.True(SpinWait.SpinUntil(() => Volatile.Read(ref pool._waitCount) == 2, TimeSpan.FromSeconds(10)), + "Second open did not wait for the creation semaphore."); + } + Assert.False(open.IsCompleted); + } + + /// + /// Prevents a stuck open from hanging the test. The caller awaits the completed task + /// separately to check whether the opens succeeded. + /// + private static async Task AssertCompletes(Task task) => + Assert.Same(task, await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(20)))); + + /// Runs synchronous opens on a dedicated thread so the test can release the login. + private static Task Open(SqlConnection connection, bool async) => + async + ? connection.OpenAsync() + : Task.Factory.StartNew(connection.Open, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + /// Owns the simulated server and a gate that blocks only its first login. + private sealed class GatedLoginServer : IDisposable + { + private readonly TdsServer _server = new(); + private readonly ManualResetEventSlim _loginEntered = new(); + private readonly ManualResetEventSlim _releaseLogin = new(); + private int _logins; + + internal int LoginCount => Volatile.Read(ref _logins); + internal string ConnectionString { get; } + + /// Starts an isolated server with room for both admitted connections. + internal GatedLoginServer() + { + _server.OnLogin7Validated = _ => + { + if (Interlocked.Increment(ref _logins) == 1) + { + _loginEntered.Set(); + Assert.True(_releaseLogin.Wait(TimeSpan.FromSeconds(15)), "Login was not released."); + } + }; + _server.Start(); + ConnectionString = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{_server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + Pooling = true, + MaxPoolSize = 2, + ConnectTimeout = 15, + }.ConnectionString; + } + + /// Waits until the first login reaches the gate. + internal void WaitForFirstLogin() => + Assert.True(_loginEntered.Wait(TimeSpan.FromSeconds(10)), "First login did not start."); + + /// Allows the first login to finish, including during failure cleanup. + internal void ReleaseFirstLogin() => _releaseLogin.Set(); + + /// Releases the gate and disposes the server before its synchronization objects. + public void Dispose() + { + ReleaseFirstLogin(); + _server.Dispose(); + _loginEntered.Dispose(); + _releaseLogin.Dispose(); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/RegisteredApplicationPoolTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/RegisteredApplicationPoolTests.cs new file mode 100644 index 0000000000..4819429b18 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/RegisteredApplicationPoolTests.cs @@ -0,0 +1,135 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Data.Common.ConnectionString; +using Microsoft.Data.ProviderBase; +using Microsoft.Data.SqlClient.ConnectionPool; +using Microsoft.SqlServer.TDS; +using Microsoft.SqlServer.TDS.Login7; +using Microsoft.SqlServer.TDS.Servers; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests; + +/// +/// Verifies registered application attribution when physical connections are pooled. +/// +[Collection(SimulatedServerTestCollection.Name)] +public class RegisteredApplicationPoolTests +{ + /// + /// Verifies application identity does not split pools and a reused physical connection keeps + /// the identity reported when that physical connection was created. + /// + [Fact] + public void DifferentRegisteredApplications_ReusePhysicalConnection_WithOriginalIdentity() + { + using TdsServer server = new(); + server.EnableUserAgentFeatureExt = true; + var payloads = new ConcurrentQueue(); + server.OnLogin7Validated = login => payloads.Enqueue(GetUserAgentPayload(login)); + server.Start(); + + string connectionString = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + Pooling = true, + }.ConnectionString; + + using var firstConnection = new SqlConnection(connectionString) + { + RegisteredApplication = RegisteredApplication.EntityFrameworkCore, + }; + + try + { + firstConnection.Open(); + firstConnection.Close(); + + using var secondConnection = new SqlConnection(connectionString) + { + RegisteredApplication = RegisteredApplication.SemanticKernel, + }; + secondConnection.Open(); + + Assert.Equal(1, server.Login7Count); + Assert.Single(payloads); + Assert.True(payloads.TryDequeue(out byte[]? payload)); + Assert.Equal(UserAgent.GetUcs2Bytes(RegisteredApplication.EntityFrameworkCore).ToArray(), payload); + } + finally + { + SqlConnection.ClearPool(firstConnection); + } + } + + /// + /// Verifies a physical connection created by channel-pool background warmup reports the + /// unknown application because no logical connection owns the creation. + /// + [Fact] + public async Task MinPoolSizeBackgroundConnection_ReportsUnknownApplication() + { + using TdsServer server = new(); + server.EnableUserAgentFeatureExt = true; + var payloads = new ConcurrentQueue(); + server.OnLogin7Validated = login => payloads.Enqueue(GetUserAgentPayload(login)); + server.Start(); + + string connectionString = new SqlConnectionStringBuilder + { + DataSource = $"localhost,{server.EndPoint.Port}", + Encrypt = SqlConnectionEncryptOption.Optional, + Pooling = true, + MinPoolSize = 1, + MaxPoolSize = 1, + }.ConnectionString; + var connectionOptions = new SqlConnectionOptions(connectionString); + var poolOptions = new DbConnectionPoolGroupOptions( + poolByIdentity: false, + minPoolSize: 1, + maxPoolSize: 1, + creationTimeout: 15, + loadBalanceTimeout: 0, + hasTransactionAffinity: true, + idleTimeout: 0); + var poolKey = new ConnectionPoolKey( + connectionString, + credential: null, + accessToken: null, + accessTokenCallback: null, + sspiContextProvider: null); + var poolGroup = new DbConnectionPoolGroup(connectionOptions, poolKey, poolOptions); + using var pool = new ChannelDbConnectionPool( + SqlConnectionFactory.Instance, + poolGroup, + DbConnectionPoolIdentity.NoIdentity, + new SqlConnectionPoolProviderInfo()); + + pool.Startup(); + await pool.WarmupLoopTask!; + + Assert.Equal(1, server.Login7Count); + Assert.Single(payloads); + Assert.True(payloads.TryDequeue(out byte[]? payload)); + Assert.Equal(UserAgent.GetUcs2Bytes(RegisteredApplication.Unknown).ToArray(), payload); + } + + /// + /// Extracts the USERAGENT extension payload from a validated LOGIN7 token. + /// + /// The validated LOGIN7 token. + /// The USERAGENT payload bytes. + private static byte[] GetUserAgentPayload(TDSLogin7Token login) => + login.FeatureExt + .OfType() + .Single(option => option.FeatureID == TDSFeatureID.UserAgentSupport) + .Data + .ToArray(); +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseDeadlockTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseDeadlockTest.cs index 465e284b6d..fa333e9215 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseDeadlockTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseDeadlockTest.cs @@ -119,7 +119,9 @@ public void CloseOrDispose_WithPendingAsyncRead_DoesNotDeadlock(bool disposeInst // pool. Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; @@ -292,7 +294,9 @@ public void CloseOrDispose_DuringPreLoginHandshake_DoesNotDeadlock(bool disposeI ConnectRetryCount = 0, Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; @@ -502,7 +506,9 @@ public void CloseOrDispose_DuringTlsHandshake_DoesNotDeadlock(bool disposeInstea ConnectRetryCount = 0, Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseHandshakeCancellationTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseHandshakeCancellationTest.cs index 9a41c2f3c7..b11b5ed291 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseHandshakeCancellationTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseHandshakeCancellationTest.cs @@ -119,7 +119,9 @@ public void CancelOpenAsyncDuringTlsHandshake_DoesNotDeadlock() ConnectRetryCount = 0, Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseRaceDeadlockTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseRaceDeadlockTest.cs index 27ab3a7516..78a099bba4 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseRaceDeadlockTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SNICloseRaceDeadlockTest.cs @@ -106,7 +106,9 @@ public void ResponseCompletionRacesClose_DoesNotDeadlock(bool disposeInsteadOfCl // connection (reaching SNIClose) instead of returning it to the pool. Pooling = false, #if NETFRAMEWORK + #pragma warning disable 618 // TransparentNetworkIPResolution is obsolete TransparentNetworkIPResolution = false, + #pragma warning restore 618 #endif }; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/TdsParserDecimalTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/TdsParserDecimalTests.cs new file mode 100644 index 0000000000..57a4f2cfd8 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/TdsParserDecimalTests.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Data; +using System.Data.SqlTypes; +using System.Globalization; +using System.Reflection; +using Microsoft.Data.SqlClient.Tests.Common; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests +{ + /// + /// Verifies decimal scale adjustment and RPC precision validation preserve representable values. + /// + [Collection(AppContextSwitchTestCollection.Name)] + public class TdsParserDecimalTests + { + /// + /// CLR and SQL zero must serialize when precision equals scale, regardless of input scale or sign. + /// + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void RpcDecimalParameter_ZeroFitsPrecisionEqualToScale(bool isSqlDecimal, bool truncate) + { + using LocalAppContextSwitchesHelper switches = new(); + switches.TruncateScaledDecimal = truncate; + + for (byte oldScale = 0; oldScale <= 28; oldScale++) + { + foreach (bool negative in new[] { false, true }) + { + decimal zero = new(0, 0, 0, negative, oldScale); + object value = isSqlDecimal ? (object)new SqlDecimal(zero) : zero; + for (byte scale = 1; scale <= 38; scale++) + { + WriteDecimalParameter(value, scale, scale); + } + } + } + } + + /// + /// Excess SqlDecimal precision must not reject signed zero or a value that rounds/truncates to zero. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RpcDecimalParameter_ZeroWithExcessPrecisionFits(bool negative) + { + using LocalAppContextSwitchesHelper switches = new(); + foreach (bool truncate in new[] { false, true }) + { + switches.TruncateScaledDecimal = truncate; + WriteDecimalParameter(new SqlDecimal(38, 3, !negative, 0, 0, 0, 0), 3, 3); + WriteDecimalParameter(SqlDecimal.ConvertToPrecScale( + new SqlDecimal(new decimal(0, 0, 0, negative, 3)), 38, 3), 3, 3); + WriteDecimalParameter(new SqlDecimal(38, 4, !negative, 1, 0, 0, 0), 3, 3); + } + } + + /// + /// Exempting zero must not permit nonzero CLR or SQL values with insufficient parameter precision. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public void RpcDecimalParameter_NonzeroExceedingPrecisionThrows(bool isSqlDecimal) + { + foreach (decimal value in new[] { -1m, 1m }) + { + object parameterValue = isSqlDecimal ? (object)new SqlDecimal(value) : value; + TargetInvocationException exception = Assert.Throws( + () => WriteDecimalParameter(parameterValue, 3, 3)); + Assert.IsType(exception.InnerException); + } + } + + /// + /// Scale adjustment must retain the rounding switch and precision needed for nonzero values. + /// + [Theory] + [InlineData("0.001", 3, "0.001", "0.001", 3)] + [InlineData("0.01", 3, "0.010", "0.010", 3)] + [InlineData("1", 3, "1.000", "1.000", 4)] + [InlineData("-1", 3, "-1.000", "-1.000", 4)] + [InlineData("1.005", 2, "1.01", "1.00", 3)] + [InlineData("-1.005", 2, "-1.01", "-1.00", 3)] + public void AdjustDecimalScale_NonzeroPreservesValueAndPrecision( + string value, int scale, string rounded, string truncated, int precision) + { + using LocalAppContextSwitchesHelper switches = new(); + foreach (bool truncate in new[] { false, true }) + { + switches.TruncateScaledDecimal = truncate; + SqlDecimal adjusted = TdsParser.AdjustDecimalScale( + decimal.Parse(value, CultureInfo.InvariantCulture), scale); + + Assert.Equal(truncate ? truncated : rounded, adjusted.ToString()); + Assert.Equal(scale, adjusted.Scale); + Assert.Equal(precision, adjusted.Precision); + } + } + + /// + /// Rescaling the CLR decimal limits must not reintroduce the overflow fixed by PR #4443. + /// + [Theory] + [InlineData(2)] + [InlineData(9)] + public void AdjustDecimalScale_LargeValuesRemainSqlDecimal(int scale) + { + foreach (decimal value in new[] { decimal.MinValue, decimal.MaxValue }) + { + SqlDecimal adjusted = TdsParser.AdjustDecimalScale(value, scale); + + Assert.Equal(value.ToString(CultureInfo.InvariantCulture) + "." + new string('0', scale), adjusted.ToString()); + Assert.Equal(scale, adjusted.Scale); + Assert.Equal(29 + scale, adjusted.Precision); + Assert.Throws(() => adjusted.Value); + } + } + + /// + /// Serializes one RPC parameter into a fresh parser buffer without connecting to a server. + /// + /// The CLR decimal or SqlDecimal parameter value. + /// The declared parameter precision. + /// The declared parameter scale. + private static void WriteDecimalParameter(object value, byte precision, byte scale) + { + const BindingFlags Flags = BindingFlags.Instance | BindingFlags.NonPublic; + TdsParser parser = new(false, false); + object state = typeof(TdsParser).GetField("_physicalStateObj", Flags)!.GetValue(parser)!; + SqlParameter parameter = new("@Value", SqlDbType.Decimal) + { + Value = value, + Precision = precision, + Scale = scale + }; + parameter.Validate(0, false); + + using SqlCommand command = new(); + MethodInfo write = typeof(TdsParser).GetMethod("TDSExecuteRPCAddParameter", Flags)!; + Assert.Null(write.Invoke(parser, new object[] + { + state, parameter, parameter.InternalMetaType, (byte)0, command, false + })); + } + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/UserAgentTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/UserAgentTests.cs index 80b826700e..fc242fa9d2 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/UserAgentTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/UserAgentTests.cs @@ -72,11 +72,12 @@ public void Value_Runtime_Parts() // // The format should be: // - // 1|MS-MDS|{Driver Version}|{Arch}|{OS Type}|{OS Info}|{Runtime Info} + // 2|MS-MDS|{Driver Version}|{Arch}|{OS Type}|{OS Info}|{Runtime Info}| + // {App Id}|{Driver Properties} // var parts = value.Split('|'); - Assert.Equal(7, parts.Length); - Assert.Equal("1", parts[0]); + Assert.Equal(9, parts.Length); + Assert.Equal("2", parts[0]); Assert.Equal("MS-MDS", parts[1]); Assert.Equal(ThisAssembly.PackageVersion, parts[2]); @@ -116,6 +117,13 @@ public void Value_Runtime_Parts() // Runtime Info must be non-empty and 44 characters or less. Assert.True(parts[6] == "Unknown" || parts[6].Length > 0); Assert.True(parts[6].Length <= 44); + + // App Id defaults to Unknown. + Assert.Equal("0", parts[7]); + + // Driver Properties is an unpadded 64-bit hexadecimal value. + Assert.InRange(parts[8].Length, 1, 16); + Assert.Matches("^[0-9A-F]+$", parts[8]); } /// @@ -149,6 +157,124 @@ public void Ucs2Bytes_Runtime_Parts() Assert.Equal(UserAgent.Value, value); } + /// + /// Test that the default payload is reused when no application identity is + /// set on the connection. + /// + [Fact] + public void GetUcs2Bytes_Unknown_App_Returns_Value() + { + var bytes = UserAgent.GetUcs2Bytes(RegisteredApplication.Unknown); + + Assert.Equal(UserAgent.Ucs2Bytes.ToArray(), bytes.ToArray()); + Assert.Equal(9, Decode(bytes).Split('|').Length); + } + + /// + /// Test that an application identity is reported in the App Id part, + /// leaving the other parts unchanged. + /// + [Fact] + public void GetUcs2Bytes_App_Sets_App_Id() + { + string value = Decode(UserAgent.GetUcs2Bytes(RegisteredApplication.SemanticKernel)); + + _output.WriteLine($"UserAgent with app: {value}"); + + var parts = value.Split('|'); + Assert.Equal(9, parts.Length); + Assert.Equal("2", parts[7]); + + // Every other part matches the default payload. + var defaultParts = UserAgent.Value.Split('|'); + for (int i = 0; i < parts.Length; ++i) + { + if (i != 7) + { + Assert.Equal(defaultParts[i], parts[i]); + } + } + } + + /// + /// Test that the payload for an application identity is built once and + /// reused across logins. + /// + [Fact] + public void GetUcs2Bytes_App_Reuses_Payload() + { + Assert.True( + UserAgent.GetUcs2Bytes(RegisteredApplication.ManagementStudio).Span.Overlaps( + UserAgent.GetUcs2Bytes(RegisteredApplication.ManagementStudio).Span)); + } + + /// + /// Test that Build() emits unpadded uppercase hexadecimal values. + /// + [Theory] + [InlineData((ushort)0, (ulong)0, "2|A|B|X64|C|D|E|0|0")] + [InlineData((ushort)7, (ulong)1, "2|A|B|X64|C|D|E|7|1")] + [InlineData((ushort)0x00AB, (ulong)0, "2|A|B|X64|C|D|E|AB|0")] + [InlineData(ushort.MaxValue, ulong.MaxValue, "2|A|B|X64|C|D|E|FFFF|FFFFFFFFFFFFFFFF")] + public void Build_App_Id_And_Driver_Properties( + ushort appId, + ulong driverProperties, + string expected) + { + Assert.Equal( + expected, + Build( + maxLen: 256, + payloadVersion: "2", + driverName: "A", + driverVersion: "B", + Architecture.X64, + osType: "C", + osInfo: "D", + runtimeInfo: "E", + appId: appId, + driverProperties: driverProperties)); + } + + /// + /// Calls the payload builder with optional defaults for the version 2 fields. + /// + /// The maximum payload length. + /// The payload version. + /// The driver name. + /// The driver version. + /// The process architecture. + /// The operating system type. + /// The operating system description. + /// The runtime description. + /// The application identifier. + /// The driver feature flags. + /// The formatted payload. + private static string Build( + ushort maxLen, + string payloadVersion, + string driverName, + string driverVersion, + Architecture architecture, + string osType, + string osInfo, + string runtimeInfo, + ushort appId = 0, + ulong driverProperties = 0) => + UserAgent.Build( + maxLen, payloadVersion, driverName, driverVersion, architecture, osType, osInfo, runtimeInfo, + appId, driverProperties); + + /// + /// Decode a UCS-2 encoded payload back to its string form. + /// + private static string Decode(ReadOnlyMemory bytes) => + #if NET + Encoding.Unicode.GetString(bytes.Span); + #else + Encoding.Unicode.GetString(bytes.ToArray()); + #endif + /// /// Test the Build() function when it truncates the overall length. /// @@ -171,11 +297,14 @@ public void Ucs2Bytes_Runtime_Parts() [InlineData(13, "2|A|B|X64|C|D")] [InlineData(14, "2|A|B|X64|C|D|")] [InlineData(15, "2|A|B|X64|C|D|E")] + [InlineData(16, "2|A|B|X64|C|D|E|")] + [InlineData(20, "2|A|B|X64|C|D|E|0|0")] + [InlineData(25, "2|A|B|X64|C|D|E|0|0")] public void Build_Truncate_Overall(ushort maxLen, string expected) { Assert.Equal( expected, - UserAgent.Build( + Build( maxLen, payloadVersion: "2", driverName: "A", @@ -195,13 +324,13 @@ public void Build_Truncate_Payload_Version() // The payload version is longer than max length. Assert.Equal( "P", - UserAgent.Build( + Build( 1, "PV", "A", "B", Architecture.X64, "C", "D", "E")); // The payload version is longer than its per-field max length of 2. Assert.Equal( - "12|A|B|X64|C|D|E", - UserAgent.Build( + "12|A|B|X64|C|D|E|0|0", + Build( 128, "1234", "A", "B", Architecture.X64, "C", "D", "E")); } @@ -214,13 +343,13 @@ public void Build_Truncate_Driver_Name() // The driver name is longer than max length. Assert.Equal( "2|DriverNa", - UserAgent.Build( + Build( 10, "2", "DriverName", "B", Architecture.X64, "C", "D", "E")); // The driver name is longer than its per-field max length of 12. Assert.Equal( - "2|LongDriverNa|B|X64|C|D|E", - UserAgent.Build( + "2|LongDriverNa|B|X64|C|D|E|0|0", + Build( 128, "2", "LongDriverName", "B", Architecture.X64, "C", "D", "E")); } @@ -234,14 +363,14 @@ public void Build_Truncate_Driver_Version() // The driver version is longer than max length. Assert.Equal( "2|A|DriverVe", - UserAgent.Build( + Build( 12, "2", "A", "DriverVersion", Architecture.X64, "C", "D", "E")); // The driver version is longer than its per-field max length of 24. Assert.Equal( - "2|A|ReallyLongDriverVersionS|X64|C|D|E", - UserAgent.Build( + "2|A|ReallyLongDriverVersionS|X64|C|D|E|0|0", + Build( 128, "2", "A", "ReallyLongDriverVersionString", Architecture.X64, "C", "D", "E")); } @@ -255,7 +384,7 @@ public void Build_Truncate_Arch() // The Architecture puts the overall length over the max. Assert.Equal( "2|A|B|Arm6", - UserAgent.Build( + Build( 10, "2", "A", "B", Architecture.Arm64, "C", "D", "E")); // There are no Architecture enum values defined in .NET Framework @@ -264,8 +393,8 @@ public void Build_Truncate_Arch() #if NET // The Architecture is longer than its per-field max length of 10. Assert.Equal( - "2|A|B|LoongArch6|C|D|E", - UserAgent.Build( + "2|A|B|LoongArch6|C|D|E|0|0", + Build( 128, "2", "A", "B", Architecture.LoongArch64, "C", "D", "E")); #endif } @@ -279,13 +408,13 @@ public void Build_Truncate_OS_Type() // The OS Type puts the overall length over the max. Assert.Equal( "2|A|B|X64|LongOs", - UserAgent.Build( + Build( 16, "2", "A", "B", Architecture.X64, "LongOsName", "D", "E")); // The OS Type is longer than its per-field max length of 10. Assert.Equal( - "2|A|B|X64|VeryLongOs|D|E", - UserAgent.Build( + "2|A|B|X64|VeryLongOs|D|E|0|0", + Build( 128, "2", "A", "B", Architecture.X64, "VeryLongOsName", "D", "E")); } @@ -299,13 +428,13 @@ public void Build_Truncate_OS_Info() // The OS Info puts the overall length over the max. Assert.Equal( "2|A|B|X64|C|LongOsI", - UserAgent.Build( + Build( 19, "2", "A", "B", Architecture.X64, "C", "LongOsInfo", "E")); // The OS Type is longer than its per-field max length of 44. Assert.Equal( - "2|A|B|X64|C|01234567890123456789012345678901234567890123|E", - UserAgent.Build( + "2|A|B|X64|C|01234567890123456789012345678901234567890123|E|0|0", + Build( 128, "2", "A", "B", Architecture.X64, "C", "01234567890123456789012345678901234567890123456789", "E")); @@ -320,14 +449,14 @@ public void Build_Truncate_Runtime_Info() // The Runtime Info puts the overall length over the max. Assert.Equal( "2|A|B|X64|C|D|LongRunt", - UserAgent.Build( + Build( 22, "2", "A", "B", Architecture.X64, "C", "D", "LongRuntimeInfo")); // The Runtime Type is longer than its per-field max length of 44. Assert.Equal( - "2|A|B|X64|C|D|01234567890123456789012345678901234567890123", - UserAgent.Build( + "2|A|B|X64|C|D|01234567890123456789012345678901234567890123|0|0", + Build( 128, "2", "A", "B", Architecture.X64, "C", "D", "01234567890123456789012345678901234567890123456789")); } @@ -340,7 +469,7 @@ public void Build_Truncate_Runtime_Info() public void Build_Truncate_Most() { var name = - UserAgent.Build( + Build( 192, // Payload version > 2 chars. "1234", @@ -357,7 +486,7 @@ public void Build_Truncate_Most() "D01234567890123456789012345678901234567890123456789", // Runtime Info > 44 chars. "E01234567890123456789012345678901234567890123456789"); - Assert.Equal(145, name.Length); + Assert.Equal(149, name.Length); Assert.Equal( "12|" + "A01234567890|" + @@ -365,7 +494,8 @@ public void Build_Truncate_Most() "X64|" + "C012345678|" + "D0123456789012345678901234567890123456789012|" + - "E0123456789012345678901234567890123456789012", + "E0123456789012345678901234567890123456789012|" + + "0|0", name); } @@ -380,7 +510,7 @@ public void Build_Truncate_Most() public void Build_Truncate_All() { var name = - UserAgent.Build( + Build( 192, // Payload version > 2 chars. "1234", @@ -396,7 +526,7 @@ public void Build_Truncate_All() "D01234567890123456789012345678901234567890123456789", // Runtime Info > 44 chars. "E01234567890123456789012345678901234567890123456789"); - Assert.Equal(152, name.Length); + Assert.Equal(156, name.Length); Assert.Equal( "12|" + "A01234567890|" + @@ -404,7 +534,8 @@ public void Build_Truncate_All() "LoongArch6|" + "C012345678|" + "D0123456789012345678901234567890123456789012|" + - "E0123456789012345678901234567890123456789012", + "E0123456789012345678901234567890123456789012|" + + "0|0", name); } #endif diff --git a/src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.default.jsonc b/src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.default.jsonc index b2200b03c0..0a34397660 100644 --- a/src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.default.jsonc +++ b/src/Microsoft.Data.SqlClient/tests/tools/Microsoft.Data.SqlClient.TestUtilities/config.default.jsonc @@ -19,6 +19,11 @@ "AADServicePrincipalSecret": "", "AzureKeyVaultURL": "", "AzureKeyVaultTenantId": "", + // True if the environment supports Entra Integrated authentication. + "SupportsEntraIntegrated": false, + // True if the environment supports Windows Integrated authentication. + // + // TODO: Default values should be empty/false. "SupportsIntegratedSecurity": true, "LocalDbAppName": "", "LocalDbSharedInstanceName": "", @@ -30,6 +35,7 @@ "IsDNSCachingSupportedCR": false, "IsDNSCachingSupportedTR": false, "EnclaveAzureDatabaseConnString": "", + // TODO: Default values should be empty/false. "ManagedIdentitySupported": true, "UserManagedIdentityClientId": "", "PowerShellPath": "", diff --git a/src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.EndPoint/TDS.EndPoint.csproj b/src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.EndPoint/TDS.EndPoint.csproj index cd68ab9016..97b98c8425 100644 --- a/src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.EndPoint/TDS.EndPoint.csproj +++ b/src/Microsoft.Data.SqlClient/tests/tools/TDS/TDS.EndPoint/TDS.EndPoint.csproj @@ -3,7 +3,6 @@ Microsoft.SqlServer.TDS.EndPoint Microsoft.SqlServer.TDS.EndPoint netstandard2.0 - $(OS) diff --git a/src/Microsoft.SqlServer.Server/Versions.props b/src/Microsoft.SqlServer.Server/Versions.props index e03f8d2967..a83b5d246a 100644 --- a/src/Microsoft.SqlServer.Server/Versions.props +++ b/src/Microsoft.SqlServer.Server/Versions.props @@ -45,13 +45,8 @@ not valid in a file version. Start with the suffix-free package base and append the configured file-version build number, which converts a three-part package version such as 1.2.3-preview1 to 1.2.3.42. - - A revision-enabled pipeline supplies a four-part package base such as 1.2.3.42. That base - is already a complete file version, so the following conditional assignment overrides the - fallback instead of producing an invalid five-part version. --> $(SqlServerPackageVersionBase).$(FileVersionBuildNumber) - $(SqlServerPackageVersionBase) @@ -60,18 +55,24 @@ When a build number is provided, the default version is used as the base of the package and file versions. - If a build suffix is provided, this is appended to the package version. This is meant to be - used by automated pre-release systems to indicate the source of the build. + If a build suffix is provided, it is appended as a prerelease tag. This is meant to be used + by automated pre-release systems to indicate the source of the build. - If a build suffix is not provided, no pre-release tag will be added to package version. If - the default version already has a pre-release tag added to it (eg, "7.0.0-preview1") this - will be retained for the package version, but will be stripped off for the file version - (letters are not allowed in file/assembly versions). This is meant to be used by official - build pipelines to generate production-ready builds. + The build number is then appended to any version carrying a prerelease tag — whether that + tag came from the declared version or from the build suffix — so every automated run + produces a distinct package version (eg, "1.1.0-preview1" becomes + "1.1.0-preview1.26238.3"). A version with no prerelease tag is left exactly as declared, + because released packages are not stamped with a build number. The tag is always stripped + for the file version (letters are not allowed in file/assembly versions). This is meant to + be used by official build pipelines to generate production-ready builds. --> - $(SqlServerNextVersion)-$(BuildSuffix)$(BuildNumber) - $(SqlServerNextVersion) + <_SqlServerCandidateVersion>$(SqlServerNextVersion) + <_SqlServerCandidateVersion Condition="'$(BuildSuffix)' != ''">$(SqlServerNextVersion)-$(BuildSuffix) + + + $(_SqlServerCandidateVersion).$(BuildNumber) + $(_SqlServerCandidateVersion) $(SqlServerNextVersion.Split('-')[0]).$(FileVersionBuildNumber) @@ -100,4 +101,11 @@ $(SqlServerFileVersion.Split('.')[0]).0.0.0 + + + + + diff --git a/tools/PackageCompatibility/NuGet.config b/tools/PackageCompatibility/NuGet.config index 1c814bbc3c..0d35132d82 100644 --- a/tools/PackageCompatibility/NuGet.config +++ b/tools/PackageCompatibility/NuGet.config @@ -10,4 +10,39 @@ --> + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/PackageCompatibility/global.json b/tools/PackageCompatibility/global.json index af3f080d1f..78c9dbe651 100644 --- a/tools/PackageCompatibility/global.json +++ b/tools/PackageCompatibility/global.json @@ -8,7 +8,7 @@ // global.json lookup stops at the nearest file and does not merge with the repo-root file. // This subtree therefore needs to repeat the root SDK settings so commands run from // tools/PackageCompatibility keep using the same SDK behavior. - "version": "10.0.300", + "version": "10.0.401", "rollForward": "patch", "allowPrerelease": false } diff --git a/tools/PackageValidator/README.md b/tools/PackageValidator/README.md index f09ded9db4..3eb3996d4d 100644 --- a/tools/PackageValidator/README.md +++ b/tools/PackageValidator/README.md @@ -28,6 +28,8 @@ For every package it opens, PackageValidator reports and validates: judged only over implementation assemblies. - **Symbols** — sibling `.snupkg` matching by debug GUID with portable-PDB checksum verification, embedded-symbol detection, and orphan/mismatch reporting. +- **Source coverage** — inspect embedded and matched portable PDBs for missing Source Link mappings, + unembedded generated/temporary sources, and non-normalized document paths, without fetching sources. - **Intrinsic rules** — severity-tagged findings for missing/mismatched symbols, unsigned/delay-signed assemblies, dependency inconsistencies, and more. - **Expected versions** — optional `--expect-*` assertions for inter-package version-match @@ -80,6 +82,9 @@ or one of the following finding **categories**: - `symbol-checksum-mismatch` - `symbol-orphan` - `symbol-duplicate` +- `missing-source-link` +- `untracked-source` +- `non-deterministic-source-path` - `delay-signed` - `unsigned` - `package-unsigned` @@ -88,6 +93,34 @@ or one of the following finding **categories**: - `unexpected-file-version` - `unexpected-assembly-version` +### Source coverage and deterministic paths + +The three source categories are warnings, and follow NuGet Package Explorer's offline checks: + +- `missing-source-link`: a document is neither embedded (Embedded Source custom debug information) + nor covered by a Source Link map. Matching is case-insensitive, supports exact keys and a single + trailing `*`, and does not download the resulting URL or verify remote source checksums. +- `untracked-source`: an **unembedded** document has no Source Link mapping, or contains an `obj`, + `temp`, or `tmp` path segment (case-insensitive, either slash separator), even when Source Link maps it. +- `non-deterministic-source-path`: a document path does not begin with `/_`, including embedded + documents. This checks path normalization, not complete build reproducibility. + +For example, `/_1/src/File.cs` is normalized but is **not** covered by a `/_/*` mapping. +Generated documents under `/_/obj/` pass when embedded. Reference and satellite assemblies without +PDBs do not receive source findings. `--no-snupkg` still checks embedded PDBs. Malformed source +metadata/maps are inspection errors (exit 1), not successful coverage checks. +Like NuGet Package Explorer, document records with nil name, language, hash algorithm, or hash +handles are not inspected. + +JSON includes per-PDB `sourceCoverage` records on each inspected binary, with document counts and +the offending paths; human-readable findings include the assembly, PDB, and document path. + +Gate these checks from the repository root (keep matching `.snupkg` files beside the packages): + +```powershell +dotnet run --project .\tools\PackageValidator\src -- --fail-on missing-source-link untracked-source non-deterministic-source-path +``` + ### `--expect-*` values Each `--expect-*` value is either `VALUE` (applies to every package in the run) or `id=VALUE` @@ -219,6 +252,8 @@ The `test/` project is an xUnit v3 suite running on Microsoft.Testing.Platform. public-key-token computation, binary classification, SemVer 2.0 range evaluation (including prerelease ordering and malformed-input rejection), the rules engine, and expected-version assertions. +Source coverage tests generate portable and embedded PDB fixtures with `MetadataBuilder`, exercising +the extra-SourceRoot mapping regression, generated sources, path normalization, and symbol matching. ```bash # From tools/PackageValidator/test diff --git a/tools/PackageValidator/global.json b/tools/PackageValidator/global.json index a4b67c1ed4..bbf9d2a793 100644 --- a/tools/PackageValidator/global.json +++ b/tools/PackageValidator/global.json @@ -8,7 +8,7 @@ // global.json lookup stops at the nearest file and does not merge with the repo-root file. // This subtree therefore needs to repeat the root SDK settings so commands run from // tools/PackageValidator keep using the same SDK behavior. - "version": "10.0.300", + "version": "10.0.401", "rollForward": "patch", "allowPrerelease": false } diff --git a/tools/PackageValidator/src/AssemblyInspector.cs b/tools/PackageValidator/src/AssemblyInspector.cs index 35a16fcc0a..9ae9ae3db7 100644 --- a/tools/PackageValidator/src/AssemblyInspector.cs +++ b/tools/PackageValidator/src/AssemblyInspector.cs @@ -85,7 +85,8 @@ public static BinaryReport Inspect(ZipArchiveEntry entry) // Read the debug directory to learn which PDB (by GUID) this assembly was built with, // whether it already carries an embedded portable PDB, and any recorded PDB checksums. - (Guid? codeViewGuid, bool hasEmbeddedPdb, List? checksums) = ReadDebugInfo(pe); + (Guid? codeViewGuid, bool hasEmbeddedPdb, List? checksums, + List? sourceCoverage) = ReadDebugInfo(pe); // File, informational, and target-framework versions are assembly-level custom // attributes, so scan for them by attribute type name. @@ -132,6 +133,7 @@ public static BinaryReport Inspect(ZipArchiveEntry entry) CodeViewGuid = codeViewGuid, Checksums = checksums, HasEmbeddedSymbols = hasEmbeddedPdb, + SourceCoverage = sourceCoverage, }; } catch (BadImageFormatException) @@ -167,12 +169,14 @@ private static SigningStatus DetermineSigningStatus(PEReader pe, bool hasPublicK /// embeds a portable PDB, and any recorded PDB checksums. /// /// The PE reader positioned over the assembly. - /// The CodeView GUID, the embedded-PDB flag, and the recorded checksums (if any). - private static (Guid? CodeViewGuid, bool HasEmbeddedPdb, List? Checksums) ReadDebugInfo(PEReader pe) + /// The CodeView GUID, embedded-PDB flag, checksums, and embedded source coverage (if any). + private static (Guid? CodeViewGuid, bool HasEmbeddedPdb, List? Checksums, + List? SourceCoverage) ReadDebugInfo(PEReader pe) { Guid? codeViewGuid = null; bool hasEmbeddedPdb = false; List? checksums = null; + List? sourceCoverage = null; foreach (DebugDirectoryEntry entry in pe.ReadDebugDirectory()) { @@ -191,6 +195,15 @@ private static (Guid? CodeViewGuid, bool HasEmbeddedPdb, List? Chec case DebugDirectoryEntryType.EmbeddedPortablePdb: hasEmbeddedPdb = true; + try + { + using MetadataReaderProvider provider = pe.ReadEmbeddedPortablePdbDebugDirectoryData(entry); + (sourceCoverage ??= []).Add(PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "embedded")); + } + catch (BadImageFormatException ex) + { + throw new InvalidDataException("Invalid embedded portable PDB.", ex); + } break; case DebugDirectoryEntryType.PdbChecksum: @@ -211,7 +224,7 @@ private static (Guid? CodeViewGuid, bool HasEmbeddedPdb, List? Chec } } - return (codeViewGuid, hasEmbeddedPdb, checksums); + return (codeViewGuid, hasEmbeddedPdb, checksums, sourceCoverage); } /// diff --git a/tools/PackageValidator/src/Models.Report.cs b/tools/PackageValidator/src/Models.Report.cs index 23979d551b..627481f0ea 100644 --- a/tools/PackageValidator/src/Models.Report.cs +++ b/tools/PackageValidator/src/Models.Report.cs @@ -221,6 +221,9 @@ internal sealed class BinaryReport /// Gets the matching PDB's path within the symbol package, if any. public string? SymbolPackageFile { get; set; } + /// Gets source coverage for embedded and matched portable PDBs, or null when none were inspected. + public List? SourceCoverage { get; set; } + /// /// Creates a report for a native or non-assembly DLL, recording its path, native version info, /// and marking it as unmanaged. @@ -236,3 +239,22 @@ internal sealed class BinaryReport NativeVersion = nativeVersion, }; } + +/// Offline source coverage and deterministic path checks for one portable PDB. +internal sealed class PdbSourceCoverage +{ + /// Gets the symbol-package entry path, or "embedded" for an embedded portable PDB. + public required string Pdb { get; init; } + + /// Gets the number of complete document records inspected, including embedded sources. + public required int DocumentCount { get; init; } + + /// Gets documents neither embedded nor covered by a Source Link mapping. + public required List MissingSourceLinkDocuments { get; init; } + + /// Gets unembedded documents without a mapping or containing an obj, temp, or tmp segment even if mapped. + public required List UntrackedDocuments { get; init; } + + /// Gets document paths that do not begin with the deterministic /_ prefix. + public required List NonNormalizedDocuments { get; init; } +} diff --git a/tools/PackageValidator/src/PortablePdb.cs b/tools/PackageValidator/src/PortablePdb.cs index 8c71c99a0b..b803b09ceb 100644 --- a/tools/PackageValidator/src/PortablePdb.cs +++ b/tools/PackageValidator/src/PortablePdb.cs @@ -3,7 +3,9 @@ // See the LICENSE file in the project root for more information. using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; using System.Security.Cryptography; +using System.Text.Json; namespace PackageValidator; @@ -19,6 +21,124 @@ internal static class PortablePdb /// The length in bytes of the PDB id (a 16-byte GUID followed by a 4-byte stamp). private const int PdbIdLength = 20; + private static readonly Guid SourceLinkKind = new("cc110556-a091-4d38-9fec-25ab9a351a6a"); + private static readonly Guid EmbeddedSourceKind = new("0e8a571b-6926-466e-b4ad-8ab04611f5fe"); + + /// Inspects document coverage without downloading sources or decompressing embedded source content. + /// The matched or embedded portable PDB metadata reader. + /// The symbol-package entry path or "embedded". + /// The source documents that fail the offline Source Link or deterministic path checks. + /// The source metadata or Source Link map is malformed. + public static PdbSourceCoverage ReadSourceCoverage(MetadataReader reader, string pdb) + { + try + { + List mappings = ReadSourceMappings(reader); + var missing = new List(); + var untracked = new List(); + var nonNormalized = new List(); + int documentCount = 0; + foreach (DocumentHandle handle in reader.Documents) + { + Document document = reader.GetDocument(handle); + if (document.Name.IsNil || document.Language.IsNil || + document.HashAlgorithm.IsNil || document.Hash.IsNil) + { + continue; + } + documentCount++; + string path = reader.GetString(document.Name); + bool embedded = reader.GetCustomDebugInformation(handle) + .Any(h => reader.GetGuid(reader.GetCustomDebugInformation(h).Kind) == EmbeddedSourceKind); + if (!embedded) + { + bool mapped = !path.Contains('*') && mappings.Any(key => key.EndsWith('*') + ? path.StartsWith(key[..^1], StringComparison.OrdinalIgnoreCase) + : string.Equals(path, key, StringComparison.OrdinalIgnoreCase)); + if (!mapped) + { + missing.Add(path); + } + if (!mapped || path.Split(['/', '\\']).Any(segment => + segment.Equals("obj", StringComparison.OrdinalIgnoreCase) || + segment.Equals("temp", StringComparison.OrdinalIgnoreCase) || + segment.Equals("tmp", StringComparison.OrdinalIgnoreCase))) + { + untracked.Add(path); + } + } + if (!path.StartsWith("/_", StringComparison.OrdinalIgnoreCase)) + { + nonNormalized.Add(path); + } + } + return new PdbSourceCoverage + { + Pdb = pdb, + DocumentCount = documentCount, + MissingSourceLinkDocuments = missing, + UntrackedDocuments = untracked, + NonNormalizedDocuments = nonNormalized, + }; + } + catch (Exception ex) when (ex is BadImageFormatException or JsonException) + { + throw new InvalidDataException($"Invalid source metadata in PDB '{pdb}'.", ex); + } + } + + /// Reads and validates module-level Source Link patterns using NPE's exact/trailing-wildcard rules. + /// The portable PDB reader. + /// Validated document path patterns; URL reachability is deliberately not checked. + private static List ReadSourceMappings(MetadataReader reader) + { + var mappings = new List(); + foreach (CustomDebugInformationHandle handle in reader.GetCustomDebugInformation( + MetadataTokens.EntityHandle(TableIndex.Module, 1))) + { + CustomDebugInformation data = reader.GetCustomDebugInformation(handle); + if (reader.GetGuid(data.Kind) != SourceLinkKind) + { + continue; + } + using JsonDocument json = JsonDocument.Parse( + reader.GetBlobBytes(data.Value), new JsonDocumentOptions { AllowTrailingCommas = true }); + if (json.RootElement.ValueKind != JsonValueKind.Object) + { + throw new InvalidDataException("Source Link must be a JSON object."); + } + foreach (JsonProperty property in json.RootElement.EnumerateObject()) + { + if (property.Name != "documents") + { + continue; + } + if (property.Value.ValueKind != JsonValueKind.Object) + { + throw new InvalidDataException("Source Link 'documents' must be an object."); + } + foreach (JsonProperty mapping in property.Value.EnumerateObject()) + { + string key = mapping.Name; + int star = key.IndexOf('*'); + if (key.Length == 0 || (star >= 0 && star != key.Length - 1) || + mapping.Value.ValueKind != JsonValueKind.String) + { + throw new InvalidDataException("Invalid Source Link document mapping."); + } + string url = mapping.Value.GetString()!; + int urlStar = url.IndexOf('*'); + if (urlStar >= 0 && (star < 0 || urlStar != url.LastIndexOf('*'))) + { + throw new InvalidDataException("Invalid Source Link URL wildcard."); + } + mappings.Add(key); + } + } + } + return mappings; + } + /// /// Reads the debug GUID from a portable PDB. /// diff --git a/tools/PackageValidator/src/SymbolResolver.cs b/tools/PackageValidator/src/SymbolResolver.cs index aa93a09d06..7145f77a45 100644 --- a/tools/PackageValidator/src/SymbolResolver.cs +++ b/tools/PackageValidator/src/SymbolResolver.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System.IO.Compression; +using System.Reflection.Metadata; namespace PackageValidator; @@ -150,7 +151,7 @@ private static void MatchSymbolPackage( asm.HasSymbolPackageSymbols = true; asm.SymbolPackageSymbolsMatch = true; asm.SymbolPackageFile = byGuid; - asm.SymbolPackageVerifiedByChecksum = VerifyMatchedChecksum(archive, byGuid, asm); + InspectMatchedPdb(archive, byGuid, asm); matchedPdbs.Add(byGuid); } else if (pdbByPathKey.TryGetValue(StripExtension(asm.Path), out string? byPath)) @@ -201,18 +202,24 @@ private static void MatchSymbolPackage( } /// - /// Re-reads a GUID-matched PDB from the symbol package on demand and verifies it against the - /// assembly's recorded PDB checksums. Loading the bytes here (rather than retaining every PDB - /// during indexing) keeps peak memory to a single PDB. + /// Re-reads a GUID-matched PDB to verify checksums and inspect source coverage. Loading bytes + /// on demand rather than retaining all PDBs during indexing keeps peak memory to one PDB. /// /// The open symbol-package archive. /// The full archive path of the matched PDB. /// The assembly whose checksums drive verification. - /// The checksum verification result, or if the entry cannot be read. - private static bool? VerifyMatchedChecksum(ZipArchive archive, string pdbFullName, BinaryReport asm) + private static void InspectMatchedPdb(ZipArchive archive, string pdbFullName, BinaryReport asm) { ZipArchiveEntry? entry = archive.GetEntry(pdbFullName); - return entry is null ? null : VerifyChecksum(asm, ReadEntry(entry)); + if (entry is null) + { + return; + } + byte[] bytes = ReadEntry(entry); + asm.SymbolPackageVerifiedByChecksum = VerifyChecksum(asm, bytes); + using var stream = new MemoryStream(bytes, writable: false); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbStream(stream); + (asm.SourceCoverage ??= []).Add(PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), pdbFullName)); } /// diff --git a/tools/PackageValidator/src/Validator.cs b/tools/PackageValidator/src/Validator.cs index 6dd738fd42..4334ce68c8 100644 --- a/tools/PackageValidator/src/Validator.cs +++ b/tools/PackageValidator/src/Validator.cs @@ -15,6 +15,9 @@ internal static class Categories public const string SymbolChecksumMismatch = "symbol-checksum-mismatch"; public const string SymbolOrphan = "symbol-orphan"; public const string SymbolDuplicate = "symbol-duplicate"; + public const string MissingSourceLink = "missing-source-link"; + public const string UntrackedSource = "untracked-source"; + public const string NonDeterministicSourcePath = "non-deterministic-source-path"; public const string DelaySigned = "delay-signed"; public const string Unsigned = "unsigned"; public const string PackageUnsigned = "package-unsigned"; @@ -30,6 +33,7 @@ internal static class Categories SymbolOrphan, SymbolDuplicate, DelaySigned, Unsigned, PackageUnsigned, DependencyInconsistency, UnexpectedPackageVersion, UnexpectedFileVersion, UnexpectedAssemblyVersion, + MissingSourceLink, UntrackedSource, NonDeterministicSourcePath, ]; } @@ -54,12 +58,44 @@ public static void Validate(PackageReport report, VersionExpectations? expectati CheckVersionConsistency(report, findings); CheckExpectedVersions(report, expectations, findings); CheckSymbols(report, findings); + CheckSourceCoverage(report, findings); CheckSigning(report, findings); CheckPackageSignature(report, findings); report.Findings = findings.Count == 0 ? null : findings; } + /// Reports source issues only for PDBs actually inspected, leaving symbolless reference/satellite assemblies alone. + private static void CheckSourceCoverage(PackageReport report, List findings) + { + foreach (BinaryReport asm in report.Binaries.Where(b => b.IsManagedAssembly && b.SourceCoverage is not null)) + { + foreach (PdbSourceCoverage coverage in asm.SourceCoverage!) + { + AddSourceFindings(coverage.MissingSourceLinkDocuments, Categories.MissingSourceLink, + "source is neither embedded nor covered by Source Link"); + AddSourceFindings(coverage.UntrackedDocuments, Categories.UntrackedSource, + "unembedded source is unmapped or contains an obj, temp, or tmp path segment"); + AddSourceFindings(coverage.NonNormalizedDocuments, Categories.NonDeterministicSourcePath, + "source path does not begin with the deterministic '/_' prefix"); + + void AddSourceFindings(List documents, string category, string message) + { + foreach (string document in documents) + { + findings.Add(new Finding + { + Severity = Severity.Warning, + Category = category, + Target = asm.Path, + Message = $"{message}: '{document}' (PDB '{coverage.Pdb}').", + }); + } + } + } + } + } + /// /// Confirms a package's version, and its assemblies' file and assembly versions, against the /// caller-supplied expected values. Pointing every package at the same expected value provides diff --git a/tools/PackageValidator/test/SourceCoverageTests.cs b/tools/PackageValidator/test/SourceCoverageTests.cs new file mode 100644 index 0000000000..0eff81804a --- /dev/null +++ b/tools/PackageValidator/test/SourceCoverageTests.cs @@ -0,0 +1,310 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Immutable; +using System.IO.Compression; +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; +using System.Reflection.PortableExecutable; +using System.Text; +using System.Text.Json; +using Xunit; + +namespace PackageValidator.Tests; + +/// Guards Source Link coverage and reproducible source paths using real portable PDB metadata. +public class SourceCoverageTests +{ + /// Checks the extra SourceRoot regression and generated-source exceptions without fetching sources. + [Theory] + [InlineData("/_1/src/File.cs", "/_/*", false, true, true, false)] + [InlineData("/_/src/File.cs", "/_/*", false, false, false, false)] + [InlineData("/_/obj/File.g.cs", null, true, false, false, false)] + [InlineData("/_/obj/File.g.cs", "/_/*", false, false, true, false)] + [InlineData("/_/temp/File.g.cs", null, false, true, true, false)] + [InlineData("/_/TMP/File.g.cs", "/_/*", false, false, true, false)] + [InlineData(@"C:\build\obj\File.g.cs", @"C:\build\*", false, false, true, true)] + [InlineData("/home/build/src/File.cs", "/home/build/*", false, false, false, true)] + [InlineData("/home/build/obj/File.cs", null, true, false, false, true)] + [InlineData("/_/object/File.cs", "/_/*", false, false, false, false)] + [InlineData("/_/src/File.cs", "/_/SRC/file.cs", false, false, false, false)] + [InlineData("/_/src/File.cs", "/_/src/Other.cs", false, true, true, false)] + [InlineData("/_/src/File.cs", "/_/SRC/*", false, false, false, false)] + [InlineData("/_/src/File.cs", "/_/sr/*", false, true, true, false)] + [InlineData("/_/src/*.cs", "/_/*", false, true, true, false)] + public void Reads_document_coverage( + string path, string? mapKey, bool embedded, bool missing, bool untracked, bool nonNormalized) + { + string? json = mapKey is null ? null : Map(mapKey); + BlobBuilder pdb = CreatePdb(path, json, embedded); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbImage(pdb.ToImmutableArray()); + + PdbSourceCoverage coverage = PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "test.pdb"); + + Assert.Equal(1, coverage.DocumentCount); + Assert.Equal(missing ? [path] : Array.Empty(), coverage.MissingSourceLinkDocuments); + Assert.Equal(untracked ? [path] : Array.Empty(), coverage.UntrackedDocuments); + Assert.Equal(nonNormalized ? [path] : Array.Empty(), coverage.NonNormalizedDocuments); + } + + /// Invalid Source Link data remains an inspection error rather than silently passing coverage. + [Theory] + [InlineData("{")] + [InlineData("[]")] + [InlineData("{\"documents\":[]}")] + [InlineData("{\"documents\":{\"\": \"https://example.invalid/file\"}}")] + [InlineData("{\"documents\":{\"/_/*/file\": \"https://example.invalid/*\"}}")] + [InlineData("{\"documents\":{\"/_/*\": \"https://example.invalid/**\"}}")] + [InlineData("{\"documents\":{\"/_/file\": \"https://example.invalid/*\"}}")] + [InlineData("{\"documents\":{\"/_/*\": 1}}")] + public void Rejects_invalid_source_maps(string json) + { + BlobBuilder pdb = CreatePdb("/_/src/File.cs", json); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbImage(pdb.ToImmutableArray()); + Assert.Throws( + () => PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "test.pdb")); + } + + /// Source map extensibility, trailing commas, and URL suffixes follow NPE matching semantics. + [Theory] + [InlineData("{\"documents\":{\"/_/*\":\"https://example.invalid/*?raw=true\"},}")] + [InlineData("{\"documents\":{\"/_/*\":\"https://example.invalid/raw\"},\"future\":true}")] + [InlineData("{\"documents\":{\"/_/src/File.cs\":\"https://example.invalid/file\"}}")] + public void Accepts_source_map_variants(string json) + { + BlobBuilder pdb = CreatePdb("/_/src/File.cs", json); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbImage(pdb.ToImmutableArray()); + Assert.Empty(PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "test.pdb").MissingSourceLinkDocuments); + } + + /// Incomplete document records are ignored just as in NPE, rather than reported as untracked sources. + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + public void Skips_incomplete_document_records(int missingHandle) + { + BlobBuilder pdb = CreatePdb("/home/build/obj/File.cs", null, missingHandle: missingHandle); + using MetadataReaderProvider provider = MetadataReaderProvider.FromPortablePdbImage(pdb.ToImmutableArray()); + PdbSourceCoverage coverage = PortablePdb.ReadSourceCoverage(provider.GetMetadataReader(), "test.pdb"); + Assert.Equal(0, coverage.DocumentCount); + Assert.Empty(coverage.MissingSourceLinkDocuments); + Assert.Empty(coverage.UntrackedDocuments); + Assert.Empty(coverage.NonNormalizedDocuments); + } + + /// Embedded PDB source coverage is inspected even when sibling symbol processing is disabled. + [Fact] + public void Embedded_pdb_reports_source_findings() + { + BlobBuilder pdb = CreatePdb("/_1/src/File.cs", Map("/_/*")); + var debug = new DebugDirectoryBuilder(); + debug.AddEmbeddedPortablePdbEntry(pdb, 0x0100); + byte[] assembly = CreateAssembly(debug); + using var stream = new MemoryStream(); + using var archive = new ZipArchive(stream, ZipArchiveMode.Update); + ZipArchiveEntry entry = archive.CreateEntry("lib/net10.0/Test.dll"); + using (Stream content = entry.Open()) + { + content.Write(assembly); + } + BinaryReport binary = AssemblyInspector.Inspect(entry); + SymbolPackageInfo symbols = SymbolResolver.Resolve("unused.nupkg", [binary], false); + PackageReport report = Package(binary, symbols); + Validator.Validate(report); + + Assert.True(binary.HasSymbols); + Finding finding = Assert.Single(report.Findings!, f => f.Category == Categories.MissingSourceLink); + Assert.Equal(binary.Path, finding.Target); + Assert.Contains("/_1/src/File.cs", finding.Message); + Assert.Equal("embedded", Assert.Single(binary.SourceCoverage!).Pdb); + } + + /// Only GUID-matched symbol-package PDBs supply source findings, including all three gate categories. + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Symbol_package_only_checks_matched_pdbs(bool matches) + { + string directory = Path.Combine(Directory.GetCurrentDirectory(), "source-coverage-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + byte[] pdb = CreatePdb(@"C:\build\obj\File.g.cs", null).ToArray(); + string packagePath = Path.Combine(directory, "Test.nupkg"); + using (ZipArchive archive = ZipFile.Open(Path.ChangeExtension(packagePath, ".snupkg"), ZipArchiveMode.Create)) + using (Stream content = archive.CreateEntry("lib/net10.0/Test.pdb").Open()) + { + content.Write(pdb); + } + var binary = new BinaryReport + { + Path = "lib/net10.0/Test.dll", + Kind = BinaryKind.Implementation, + IsManagedAssembly = true, + CodeViewGuid = matches ? PortablePdb.TryReadGuid(pdb) : Guid.NewGuid(), + }; + PackageReport report = Package(binary, SymbolResolver.Resolve(packagePath, [binary], true)); + Validator.Validate(report); + + string[] sourceCategories = + [Categories.MissingSourceLink, Categories.UntrackedSource, Categories.NonDeterministicSourcePath]; + Assert.All(sourceCategories, category => + { + Assert.Contains(category, Categories.All); + Assert.Equal(matches, report.Findings!.Any(f => f.Category == category)); + }); + Assert.Equal(matches, binary.SourceCoverage is { Count: 1 }); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + /// Preserves negative and corrected SourceRoot baselines through real nupkg/snupkg archive inspection. + [Theory] + [InlineData("/_1/src/File.cs", true)] + [InlineData("/_/src/File.cs", false)] + public void Package_inspection_detects_extra_source_root(string document, bool missing) + { + string directory = Path.Combine(Directory.GetCurrentDirectory(), "source-root-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + try + { + byte[] pdb = CreatePdb(document, Map("/_/*")).ToArray(); + var debug = new DebugDirectoryBuilder(); + debug.AddCodeViewEntry("Test.pdb", new BlobContentId(PortablePdb.TryReadGuid(pdb)!.Value, 0), 0x0100); + string packagePath = Path.Combine(directory, "Test.nupkg"); + using (ZipArchive archive = ZipFile.Open(packagePath, ZipArchiveMode.Create)) + { + using (Stream content = archive.CreateEntry("lib/net10.0/Test.dll").Open()) + { + content.Write(CreateAssembly(debug)); + } + using var manifest = new StreamWriter(archive.CreateEntry("Test.nuspec").Open()); + manifest.Write("Test1.0.0"); + } + using (ZipArchive archive = ZipFile.Open(Path.ChangeExtension(packagePath, ".snupkg"), ZipArchiveMode.Create)) + using (Stream content = archive.CreateEntry("lib/net10.0/Test.pdb").Open()) + { + content.Write(pdb); + } + + PackageReport report = PackageInspector.Inspect(packagePath, processSnupkg: true); + Validator.Validate(report); + + BinaryReport binary = Assert.Single(report.Binaries); + Assert.True(binary.SymbolPackageSymbolsMatch); + Assert.True(binary.HasSymbols); + Assert.Equal(missing, report.Findings!.Any(f => f.Category == Categories.MissingSourceLink)); + Assert.Equal(missing, report.Findings!.Any(f => f.Category == Categories.UntrackedSource)); + Assert.DoesNotContain(report.Findings!, f => + f.Category == Categories.NonDeterministicSourcePath); + if (missing) + { + Assert.Contains(document, Assert.Single( + report.Findings!, f => f.Category == Categories.MissingSourceLink).Message); + } + } + finally + { + Directory.Delete(directory, recursive: true); + } + } + + /// Reference and satellite assemblies with no PDB must not create source-coverage findings. + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Legitimate_symbolless_assemblies_are_exempt(bool reference) + { + var binary = new BinaryReport + { + Path = "ref/net10.0/Test.dll", + Kind = reference ? BinaryKind.Reference : BinaryKind.Satellite, + IsManagedAssembly = true, + }; + PackageReport report = Package(binary, SymbolResolver.Resolve("unused.nupkg", [binary], false)); + Validator.Validate(report); + Assert.Null(report.Findings); + } + + /// Serializes an exact or wildcard map with an inert URL, without network access. + /// The source path pattern. + /// Source Link JSON containing one mapping. + private static string Map(string key) => JsonSerializer.Serialize(new + { + documents = new Dictionary + { + [key] = key.EndsWith('*') ? "https://example.invalid/*" : "https://example.invalid/file", + }, + }); + + /// Builds an in-memory portable PDB with a document and optional module/document custom debug records. + /// The document path. + /// Source Link JSON, or null to omit the module record. + /// Whether to embed the document's source. + /// A document handle to omit (1: name, 2: language, 3: hash algorithm, 4: hash). + /// The serialized portable PDB. + private static BlobBuilder CreatePdb(string path, string? json, bool embedded = false, int missingHandle = 0) + { + var metadata = new MetadataBuilder(); + DocumentHandle document = metadata.AddDocument( + missingHandle == 1 ? default : metadata.GetOrAddDocumentName(path), + missingHandle == 3 ? default : metadata.GetOrAddGuid(new Guid("8829d00f-11b8-4213-878b-770e8597ac16")), + missingHandle == 4 ? default : metadata.GetOrAddBlob(System.Security.Cryptography.SHA256.HashData("//"u8)), + missingHandle == 2 ? default : metadata.GetOrAddGuid(new Guid("3f5162f8-07c6-11d3-9053-00c04fa302a1"))); + if (json is not null) + { + metadata.AddCustomDebugInformation( + MetadataTokens.EntityHandle(TableIndex.Module, 1), + metadata.GetOrAddGuid(new Guid("cc110556-a091-4d38-9fec-25ab9a351a6a")), + metadata.GetOrAddBlob(Encoding.UTF8.GetBytes(json))); + } + if (embedded) + { + metadata.AddCustomDebugInformation( + document, + metadata.GetOrAddGuid(new Guid("0e8a571b-6926-466e-b4ad-8ab04611f5fe")), + metadata.GetOrAddBlob(new byte[] { 0, 0, 0, 0, 47, 47 })); + } + var rows = new int[MetadataTokens.TableCount]; + rows[(int)TableIndex.Module] = 1; + var builder = new PortablePdbBuilder(metadata, rows.ToImmutableArray(), default); + var result = new BlobBuilder(); + builder.Serialize(result); + return result; + } + + /// Builds a metadata-only assembly with the supplied debug directory for embedded-PDB integration testing. + /// The PE debug directory to include. + /// The serialized PE image. + private static byte[] CreateAssembly(DebugDirectoryBuilder debug) + { + var metadata = new MetadataBuilder(); + metadata.AddModule(0, metadata.GetOrAddString("Test.dll"), metadata.GetOrAddGuid(Guid.NewGuid()), default, default); + metadata.AddAssembly(metadata.GetOrAddString("Test"), new Version(1, 0, 0, 0), default, default, default, AssemblyHashAlgorithm.None); + var builder = new ManagedPEBuilder( + new PEHeaderBuilder(imageCharacteristics: Characteristics.Dll | Characteristics.ExecutableImage), + new MetadataRootBuilder(metadata), new BlobBuilder(), debugDirectoryBuilder: debug); + var result = new BlobBuilder(); + builder.Serialize(result); + return result.ToArray(); + } + + /// Wraps an inspected binary for validation with unrelated package-signing findings suppressed. + /// The inspected assembly. + /// Its resolved symbols. + /// A package report ready for validation. + private static PackageReport Package(BinaryReport binary, SymbolPackageInfo symbols) => new() + { + PackageFile = "Test.nupkg", + IsSigned = true, + Binaries = [binary], + SymbolPackage = symbols, + }; +}