From f3f3e75612138f94909f3a840df08f1ba5c2e4e8 Mon Sep 17 00:00:00 2001 From: ChrisRackauckas-Claude Date: Sun, 14 Jun 2026 05:12:06 -0400 Subject: [PATCH] Canonicalize tests to @safetestset for module isolation Convert each independent test unit to run in its own module via @safetestset, matching the canonical SciML structure (isolation between tests + world-age safety). Core (runtests.jl): - The 4 inline @testset units (Version bumping, Project file handling, Repository processing, Basic functionality legacy) become inline @safetestset blocks, each carrying the using lines it needs (OrgMaintenanceScripts, Test, plus TOML where used). - Each per-file unit (formatting, min_version_fixer, version_check_finder, invalidation_analysis, import_timing_analysis, explicit_imports_fixer, documentation_cleanup) is now @safetestset "Name" begin include("x.jl") end. The included files are made self-contained: stale "already loaded by runtests.jl" comments replaced with explicit using lines (OrgMaintenanceScripts, Test, plus Dates/TOML where the body uses now()/DateTime or TOML.print). These previously relied on Main-leaked imports from runtests.jl. - The commented-out multiprocess_testing include is preserved (still disabled), now shown in @safetestset form. QA (qa.jl): the two plain @testset units (Aqua, JET) extracted into qa_aqua.jl / qa_jet.jl and wrapped as @safetestset ... include(...). The include() form is required for the JET unit because @test_opt is a package macro that must be resolved after `using JET` runs (top-level statements in the included file macroexpand one at a time). @test_broken lines preserved. Deps: SafeTestsets added to Project.toml [extras]/[targets].test and [compat] SafeTestsets = "0.1, 1"; same UUID + compat added to test/qa/Project.toml (QA runs in its own activated env). using SafeTestsets added at top of runtests.jl. GROUP-dispatch ladder, run path, assertions, and test counts unchanged. Verified locally: GROUP=Core Pkg.test passes (Version bumping 5, Project file handling 6, Repository processing 8, Basic functionality 2, Formatting 9, Min Version Fixer 25, Version Check Finder 23, Invalidation 36, Import Timing 56, Explicit Imports 10, Documentation Cleanup 41). Co-Authored-By: Chris Rackauckas Co-Authored-By: Claude Opus 4.8 (1M context) --- Project.toml | 4 +- test/documentation_cleanup_tests.jl | 3 +- test/explicit_imports_fixer_tests.jl | 4 +- test/formatting_tests.jl | 3 +- test/import_timing_analysis_tests.jl | 4 +- test/invalidation_analysis_tests.jl | 4 +- test/min_version_fixer_tests.jl | 3 +- test/qa.jl | 76 +-------- test/qa/Project.toml | 2 + test/qa_aqua.jl | 13 ++ test/qa_jet.jl | 64 ++++++++ test/runtests.jl | 237 +++++++++++++++------------ test/version_check_finder_tests.jl | 3 +- 13 files changed, 237 insertions(+), 183 deletions(-) create mode 100644 test/qa_aqua.jl create mode 100644 test/qa_jet.jl diff --git a/Project.toml b/Project.toml index 922e6eb..22b2ae9 100644 --- a/Project.toml +++ b/Project.toml @@ -28,6 +28,7 @@ JET = "0.9, 0.10, 0.11" JSON3 = "1" JuliaFormatter = "2" LocalRegistry = "0.5" +SafeTestsets = "0.1, 1" SnoopCompileCore = "3" Statistics = "1" TOML = "1" @@ -37,7 +38,8 @@ julia = "1.10" [extras] JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" +SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["JET", "Test"] +test = ["JET", "SafeTestsets", "Test"] diff --git a/test/documentation_cleanup_tests.jl b/test/documentation_cleanup_tests.jl index b03c8d0..6cece8b 100644 --- a/test/documentation_cleanup_tests.jl +++ b/test/documentation_cleanup_tests.jl @@ -1,4 +1,5 @@ -# Note: OrgMaintenanceScripts is already loaded by runtests.jl +using OrgMaintenanceScripts +using Test @testset "Documentation Cleanup Tests" begin diff --git a/test/explicit_imports_fixer_tests.jl b/test/explicit_imports_fixer_tests.jl index 62e14bf..9274ccb 100644 --- a/test/explicit_imports_fixer_tests.jl +++ b/test/explicit_imports_fixer_tests.jl @@ -1,4 +1,6 @@ -# Note: OrgMaintenanceScripts is already loaded by runtests.jl +using OrgMaintenanceScripts +using Test +using TOML @testset "Explicit Imports Fixer" begin @testset "parse_explicit_imports_output" begin diff --git a/test/formatting_tests.jl b/test/formatting_tests.jl index eca4eef..735bbc8 100644 --- a/test/formatting_tests.jl +++ b/test/formatting_tests.jl @@ -1,4 +1,5 @@ -# Note: OrgMaintenanceScripts is already loaded by runtests.jl +using OrgMaintenanceScripts +using Test using Pkg @testset "Formatting Functions" begin diff --git a/test/import_timing_analysis_tests.jl b/test/import_timing_analysis_tests.jl index 07860ea..cef234c 100644 --- a/test/import_timing_analysis_tests.jl +++ b/test/import_timing_analysis_tests.jl @@ -1,4 +1,6 @@ -# Note: OrgMaintenanceScripts is already loaded by runtests.jl +using OrgMaintenanceScripts +using Test +using Dates @testset "Import Timing Analysis Tests" begin # Create a simple test package structure diff --git a/test/invalidation_analysis_tests.jl b/test/invalidation_analysis_tests.jl index 68bbc8e..e8b1dcd 100644 --- a/test/invalidation_analysis_tests.jl +++ b/test/invalidation_analysis_tests.jl @@ -1,4 +1,6 @@ -# Note: OrgMaintenanceScripts is already loaded by runtests.jl +using OrgMaintenanceScripts +using Test +using Dates @testset "Invalidation Analysis Tests" begin # Create a simple test package structure diff --git a/test/min_version_fixer_tests.jl b/test/min_version_fixer_tests.jl index cdf7822..9b292f5 100644 --- a/test/min_version_fixer_tests.jl +++ b/test/min_version_fixer_tests.jl @@ -1,4 +1,5 @@ -# Note: OrgMaintenanceScripts is already loaded by runtests.jl +using OrgMaintenanceScripts +using Test @testset "Minimum Version Fixer Tests" begin # Test helper functions diff --git a/test/qa.jl b/test/qa.jl index 3e266b1..915d04c 100644 --- a/test/qa.jl +++ b/test/qa.jl @@ -2,78 +2,12 @@ # Runs in the isolated test/qa environment (see test/qa/Project.toml), gated on # GROUP == "QA" in runtests.jl. -using OrgMaintenanceScripts -using Test -using Aqua -using JET -using Dates +using SafeTestsets -@testset "Aqua quality assurance" begin - # stale_deps and deps_compat are known failing and marked @test_broken below. - # Tracked in https://github.com/SciML/OrgMaintenanceScripts.jl/issues/60 - Aqua.test_all(OrgMaintenanceScripts; stale_deps = false, deps_compat = false) - # Aqua stale deps: SnoopCompileCore, YAML — tracked in https://github.com/SciML/OrgMaintenanceScripts.jl/issues/60 - @test_broken false - # Aqua deps_compat: missing compat for Dates, Distributed, LibGit2, Logging, Pkg, Printf, Random — tracked in https://github.com/SciML/OrgMaintenanceScripts.jl/issues/60 - @test_broken false +@safetestset "Aqua quality assurance" begin + include("qa_aqua.jl") end -@testset "JET static analysis" begin - @testset "Project utilities" begin - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.find_all_project_tomls(".") - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.get_project_info("Project.toml") - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.is_subpackage(".", ".") - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.get_relative_project_path(".", ".") - end - - @testset "Version check finder" begin - mktempdir() do tmpdir - test_file = joinpath(tmpdir, "test.jl") - write( - test_file, """ - # Test file with version checks - @static if VERSION >= v"1.6" - # Some code for Julia 1.6+ - end - if VERSION < v"1.9" - # Some code for older Julia - end - """ - ) - - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.find_version_checks_in_file( - test_file - ) - end - - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.find_version_checks_in_repo(".") - end - - @testset "Version bumping functions" begin - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.bump_minor_version( - "1.0.0" - ) - end - - @testset "Struct constructors" begin - @test_opt OrgMaintenanceScripts.VersionCheck("test.jl", 1, "test line", v"1.0", "v\"1.0\"") - @test_opt OrgMaintenanceScripts.InvalidationEntry("method", "file.jl", 1, "pkg", "reason", 0, 0) - @test_opt OrgMaintenanceScripts.InvalidationReport( - "repo", 0, OrgMaintenanceScripts.InvalidationEntry[], String[], Dates.now(), "summary", String[] - ) - end - - @testset "Report functions" begin - checks = OrgMaintenanceScripts.VersionCheck[] - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.print_version_check_summary( - devnull, checks - ) - - mktempdir() do tmpdir - output_file = joinpath(tmpdir, "output.jl") - @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.write_version_checks_to_script( - checks, output_file - ) - end - end +@safetestset "JET static analysis" begin + include("qa_jet.jl") end diff --git a/test/qa/Project.toml b/test/qa/Project.toml index 5c00c49..5cbb77f 100644 --- a/test/qa/Project.toml +++ b/test/qa/Project.toml @@ -3,11 +3,13 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b" OrgMaintenanceScripts = "87d49508-7ad8-40c6-ae47-b3ac92269cd4" +SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] Aqua = "0.8" JET = "0.9, 0.10, 0.11" +SafeTestsets = "0.1, 1" Test = "1" julia = "1.10" diff --git a/test/qa_aqua.jl b/test/qa_aqua.jl new file mode 100644 index 0000000..84920f8 --- /dev/null +++ b/test/qa_aqua.jl @@ -0,0 +1,13 @@ +using OrgMaintenanceScripts +using Test +using Aqua + +@testset "Aqua quality assurance" begin + # stale_deps and deps_compat are known failing and marked @test_broken below. + # Tracked in https://github.com/SciML/OrgMaintenanceScripts.jl/issues/60 + Aqua.test_all(OrgMaintenanceScripts; stale_deps = false, deps_compat = false) + # Aqua stale deps: SnoopCompileCore, YAML — tracked in https://github.com/SciML/OrgMaintenanceScripts.jl/issues/60 + @test_broken false + # Aqua deps_compat: missing compat for Dates, Distributed, LibGit2, Logging, Pkg, Printf, Random — tracked in https://github.com/SciML/OrgMaintenanceScripts.jl/issues/60 + @test_broken false +end diff --git a/test/qa_jet.jl b/test/qa_jet.jl new file mode 100644 index 0000000..36f5457 --- /dev/null +++ b/test/qa_jet.jl @@ -0,0 +1,64 @@ +using OrgMaintenanceScripts +using Test +using JET +using Dates + +@testset "JET static analysis" begin + @testset "Project utilities" begin + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.find_all_project_tomls(".") + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.get_project_info("Project.toml") + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.is_subpackage(".", ".") + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.get_relative_project_path(".", ".") + end + + @testset "Version check finder" begin + mktempdir() do tmpdir + test_file = joinpath(tmpdir, "test.jl") + write( + test_file, """ + # Test file with version checks + @static if VERSION >= v"1.6" + # Some code for Julia 1.6+ + end + if VERSION < v"1.9" + # Some code for older Julia + end + """ + ) + + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.find_version_checks_in_file( + test_file + ) + end + + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.find_version_checks_in_repo(".") + end + + @testset "Version bumping functions" begin + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.bump_minor_version( + "1.0.0" + ) + end + + @testset "Struct constructors" begin + @test_opt OrgMaintenanceScripts.VersionCheck("test.jl", 1, "test line", v"1.0", "v\"1.0\"") + @test_opt OrgMaintenanceScripts.InvalidationEntry("method", "file.jl", 1, "pkg", "reason", 0, 0) + @test_opt OrgMaintenanceScripts.InvalidationReport( + "repo", 0, OrgMaintenanceScripts.InvalidationEntry[], String[], Dates.now(), "summary", String[] + ) + end + + @testset "Report functions" begin + checks = OrgMaintenanceScripts.VersionCheck[] + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.print_version_check_summary( + devnull, checks + ) + + mktempdir() do tmpdir + output_file = joinpath(tmpdir, "output.jl") + @test_opt target_modules = (OrgMaintenanceScripts,) OrgMaintenanceScripts.write_version_checks_to_script( + checks, output_file + ) + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 242e23f..bde96b9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,7 @@ using Pkg using OrgMaintenanceScripts using Test +using SafeTestsets using TOML using Dates @@ -14,137 +15,165 @@ if GROUP == "QA" end if GROUP == "All" || GROUP == "Core" - @testset "OrgMaintenanceScripts.jl" begin - @testset "Version bumping" begin - # Test bump_minor_version - @test OrgMaintenanceScripts.bump_minor_version("1.2.3") == "1.3.0" - @test OrgMaintenanceScripts.bump_minor_version("0.1.0") == "0.2.0" - @test OrgMaintenanceScripts.bump_minor_version("2.10.5") == "2.11.0" - - # Test invalid version format - @test_throws ErrorException OrgMaintenanceScripts.bump_minor_version("1.2") - @test_throws ErrorException OrgMaintenanceScripts.bump_minor_version("1.2.3.4") - end - - @testset "Project file handling" begin - # Create a temporary test project - mktempdir() do tmpdir - project_path = joinpath(tmpdir, "Project.toml") + @safetestset "Version bumping" begin + using OrgMaintenanceScripts + using Test + + # Test bump_minor_version + @test OrgMaintenanceScripts.bump_minor_version("1.2.3") == "1.3.0" + @test OrgMaintenanceScripts.bump_minor_version("0.1.0") == "0.2.0" + @test OrgMaintenanceScripts.bump_minor_version("2.10.5") == "2.11.0" + + # Test invalid version format + @test_throws ErrorException OrgMaintenanceScripts.bump_minor_version("1.2") + @test_throws ErrorException OrgMaintenanceScripts.bump_minor_version("1.2.3.4") + end - # Test with valid Project.toml - project_data = Dict( - "name" => "TestPackage", - "uuid" => "12345678-1234-1234-1234-123456789012", - "version" => "0.1.0" - ) + @safetestset "Project file handling" begin + using OrgMaintenanceScripts + using Test + using TOML - open(project_path, "w") do io - TOML.print(io, project_data) - end + # Create a temporary test project + mktempdir() do tmpdir + project_path = joinpath(tmpdir, "Project.toml") - result = OrgMaintenanceScripts.update_project_version(project_path) - @test !isnothing(result) - @test result[1] == "0.1.0" - @test result[2] == "0.2.0" + # Test with valid Project.toml + project_data = Dict( + "name" => "TestPackage", + "uuid" => "12345678-1234-1234-1234-123456789012", + "version" => "0.1.0" + ) - # Verify file was updated - updated_project = TOML.parsefile(project_path) - @test updated_project["version"] == "0.2.0" + open(project_path, "w") do io + TOML.print(io, project_data) + end - # Test with missing version field - delete!(updated_project, "version") - open(project_path, "w") do io - TOML.print(io, updated_project) - end + result = OrgMaintenanceScripts.update_project_version(project_path) + @test !isnothing(result) + @test result[1] == "0.1.0" + @test result[2] == "0.2.0" - result = OrgMaintenanceScripts.update_project_version(project_path) - @test isnothing(result) + # Verify file was updated + updated_project = TOML.parsefile(project_path) + @test updated_project["version"] == "0.2.0" - # Test with non-existent file - result = OrgMaintenanceScripts.update_project_version(joinpath(tmpdir, "nonexistent.toml")) - @test isnothing(result) + # Test with missing version field + delete!(updated_project, "version") + open(project_path, "w") do io + TOML.print(io, updated_project) end + + result = OrgMaintenanceScripts.update_project_version(project_path) + @test isnothing(result) + + # Test with non-existent file + result = OrgMaintenanceScripts.update_project_version(joinpath(tmpdir, "nonexistent.toml")) + @test isnothing(result) end + end - @testset "Repository processing" begin - # Create a mock repository structure - mktempdir() do tmpdir - # Main Project.toml - main_project = Dict( - "name" => "MainPackage", - "uuid" => "12345678-1234-1234-1234-123456789012", - "version" => "1.0.0" - ) - open(joinpath(tmpdir, "Project.toml"), "w") do io - TOML.print(io, main_project) - end + @safetestset "Repository processing" begin + using OrgMaintenanceScripts + using Test + using TOML + + # Create a mock repository structure + mktempdir() do tmpdir + # Main Project.toml + main_project = Dict( + "name" => "MainPackage", + "uuid" => "12345678-1234-1234-1234-123456789012", + "version" => "1.0.0" + ) + open(joinpath(tmpdir, "Project.toml"), "w") do io + TOML.print(io, main_project) + end - # Create lib directory with subpackages - lib_dir = joinpath(tmpdir, "lib") - mkpath(lib_dir) - - for (i, pkg) in enumerate(["SubPkgA", "SubPkgB"]) - pkg_dir = joinpath(lib_dir, pkg) - mkpath(pkg_dir) - - sub_project = Dict( - "name" => pkg, - "uuid" => "12345678-1234-1234-1234-12345678901$i", - "version" => "0.$i.0" - ) - open(joinpath(pkg_dir, "Project.toml"), "w") do io - TOML.print(io, sub_project) - end - end + # Create lib directory with subpackages + lib_dir = joinpath(tmpdir, "lib") + mkpath(lib_dir) + + for (i, pkg) in enumerate(["SubPkgA", "SubPkgB"]) + pkg_dir = joinpath(lib_dir, pkg) + mkpath(pkg_dir) - # Initialize git repo - cd(tmpdir) do - run(`git init`) - run(`git config user.name "Test User"`) - run(`git config user.email "test@example.com"`) - run(`git add .`) - run(`git commit -m "Initial commit"`) + sub_project = Dict( + "name" => pkg, + "uuid" => "12345678-1234-1234-1234-12345678901$i", + "version" => "0.$i.0" + ) + open(joinpath(pkg_dir, "Project.toml"), "w") do io + TOML.print(io, sub_project) end + end - # Test bump_and_register_repo - result = bump_and_register_repo(tmpdir) + # Initialize git repo + cd(tmpdir) do + run(`git init`) + run(`git config user.name "Test User"`) + run(`git config user.email "test@example.com"`) + run(`git add .`) + run(`git commit -m "Initial commit"`) + end - @test !isnothing(result) - # In test environment, registration will fail since there's no real registry - # So we expect all packages to be in the failed list - @test basename(tmpdir) in result.failed || basename(tmpdir) in result.registered - @test "SubPkgA" in result.failed || "SubPkgA" in result.registered - @test "SubPkgB" in result.failed || "SubPkgB" in result.registered - # Either all succeed or all fail (in tests, they'll all fail) - @test isempty(result.registered) || isempty(result.failed) + # Test bump_and_register_repo + result = bump_and_register_repo(tmpdir) - # Verify versions were bumped - main_updated = TOML.parsefile(joinpath(tmpdir, "Project.toml")) - @test main_updated["version"] == "1.1.0" + @test !isnothing(result) + # In test environment, registration will fail since there's no real registry + # So we expect all packages to be in the failed list + @test basename(tmpdir) in result.failed || basename(tmpdir) in result.registered + @test "SubPkgA" in result.failed || "SubPkgA" in result.registered + @test "SubPkgB" in result.failed || "SubPkgB" in result.registered + # Either all succeed or all fail (in tests, they'll all fail) + @test isempty(result.registered) || isempty(result.failed) - subA_updated = TOML.parsefile(joinpath(lib_dir, "SubPkgA", "Project.toml")) - @test subA_updated["version"] == "0.2.0" + # Verify versions were bumped + main_updated = TOML.parsefile(joinpath(tmpdir, "Project.toml")) + @test main_updated["version"] == "1.1.0" - subB_updated = TOML.parsefile(joinpath(lib_dir, "SubPkgB", "Project.toml")) - @test subB_updated["version"] == "0.3.0" - end - end + subA_updated = TOML.parsefile(joinpath(lib_dir, "SubPkgA", "Project.toml")) + @test subA_updated["version"] == "0.2.0" - @testset "Basic functionality (legacy)" begin - # Test deprecated functions still exist but warn - @test_logs (:warn,) OrgMaintenanceScripts.update_manifests() - @test_logs (:warn,) OrgMaintenanceScripts.update_project_tomls() + subB_updated = TOML.parsefile(joinpath(lib_dir, "SubPkgB", "Project.toml")) + @test subB_updated["version"] == "0.3.0" end + end + @safetestset "Basic functionality (legacy)" begin + using OrgMaintenanceScripts + using Test + + # Test deprecated functions still exist but warn + @test_logs (:warn,) OrgMaintenanceScripts.update_manifests() + @test_logs (:warn,) OrgMaintenanceScripts.update_project_tomls() + end + + @safetestset "Formatting Functions" begin include("formatting_tests.jl") + end + @safetestset "Minimum Version Fixer Tests" begin include("min_version_fixer_tests.jl") + end + @safetestset "Version Check Finder Tests" begin include("version_check_finder_tests.jl") + end + @safetestset "Invalidation Analysis Tests" begin include("invalidation_analysis_tests.jl") + end + @safetestset "Import Timing Analysis Tests" begin include("import_timing_analysis_tests.jl") + end + @safetestset "Explicit Imports Fixer" begin include("explicit_imports_fixer_tests.jl") - # Temporarily commented out due to syntax error in multiprocess_testing.jl - # include("multiprocess_testing_tests.jl") + end + # Temporarily commented out due to syntax error in multiprocess_testing.jl + # @safetestset "Multiprocess Testing" begin + # include("multiprocess_testing_tests.jl") + # end + @safetestset "Documentation Cleanup Tests" begin include("documentation_cleanup_tests.jl") - # JET/Aqua static analysis runs in the QA group (GROUP=QA -> test/qa.jl). end + # JET/Aqua static analysis runs in the QA group (GROUP=QA -> test/qa.jl). end diff --git a/test/version_check_finder_tests.jl b/test/version_check_finder_tests.jl index ebc5423..f529a57 100644 --- a/test/version_check_finder_tests.jl +++ b/test/version_check_finder_tests.jl @@ -1,4 +1,5 @@ -# Note: OrgMaintenanceScripts is already loaded by runtests.jl +using OrgMaintenanceScripts +using Test @testset "Version Check Finder Tests" begin # Create temporary test files