Skip to content

Run previously failed tests first #138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
christiangnrd wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Run previously failed tests first #138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a9b6f3f
Run previously failed tests first
christiangnrd Jun 24, 2026
5149695
Update test/runtests.jl
christiangnrd Jul 15, 2026
d29a9be
Fixup
christiangnrd Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 22 additions & 8 deletions src/ParallelTestRunner.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -510,6 +510,18 @@ function default_njobs(; cpu_threads = Sys.CPU_THREADS, free_memory = available_
end

# Historical test duration database
struct TestHistoryEntry <: AbstractFloat
duration::Float64
failed::Bool
end
# successful tests are sorted before failed ones, so they always run first
Base.isless(a::TestHistoryEntry, b::TestHistoryEntry) = a.failed == b.failed ? a.duration < b.duration : a.failed < b.failed
Base.promote_rule(::Type{T}, ::Type{TestHistoryEntry}) where {T} = promote_type(T, Float64)
Base.promote_rule(::Type{TestHistoryEntry}, ::Type{T}) where {T} = promote_type(Float64,T)
# for compatibility with older versions of ParallelTestRunner
Base.convert(::Type{TestHistoryEntry}, duration::Number) = TestHistoryEntry(duration, false)
Base.convert(::Type{Float64}, entry::TestHistoryEntry) = entry.duration

function get_history_file(mod::Module)
scratch_dir = @get_scratch!("durations")
return joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)", "$(nameof(mod)).jls")
Expand All @@ -518,16 +530,17 @@ function load_test_history(mod::Module)
history_file = get_history_file(mod)
if isfile(history_file)
try
return deserialize(history_file)
hist::Dict{String, TestHistoryEntry} = deserialize(history_file)
return hist
catch e
@warn "Failed to load test history from $history_file" exception=e
return Dict{String, Float64}()
return Dict{String, TestHistoryEntry}()
end
else
return Dict{String, Float64}()
return Dict{String, TestHistoryEntry}()
end
end
function save_test_history(mod::Module, history::Dict{String, Float64})
function save_test_history(mod::Module, history::Dict{String, TestHistoryEntry})
history_file = get_history_file(mod)
try
mkpath(dirname(history_file))
Expand Down Expand Up @@ -944,7 +957,7 @@ function runtests(mod::Module, args::ParsedArgs;
tests = collect(keys(testsuite))
Random.shuffle!(tests)
historical_durations = load_test_history(mod)
sort!(tests, by = x -> -get(historical_durations, x, Inf))
sort!(tests, by = x -> get(historical_durations, x, Inf), rev = true)

return _runtests(
mod, args;
Expand All @@ -969,7 +982,7 @@ end
function _runtests(mod::Module, args::ParsedArgs;
testsuite::Dict{String,Expr} = find_tests(pwd()),
tests::Vector{String},
historical_durations::Dict{String, Float64},
historical_durations::Dict{String, TestHistoryEntry},
init_code = :(),
init_worker_code = :(),
test_worker = Returns(nothing),
Expand Down Expand Up @@ -1084,7 +1097,7 @@ function _runtests(mod::Module, args::ParsedArgs;
## currently-running
for (test, start_time) in running_snapshot
elapsed = time() - start_time
duration = get(historical_durations, test, est_per_test)
duration::Float64 = get(historical_durations, test, est_per_test)
est_remaining += max(0.0, duration - elapsed)
end
## yet-to-run
Expand Down Expand Up @@ -1401,7 +1414,7 @@ function _runtests(mod::Module, args::ParsedArgs;

if result isa AbstractTestRecord
testset = result[]::DefaultTestSet
historical_durations[testname] = stop - start
historical_durations[testname] = TestHistoryEntry(stop - start, anynonpass(testset))
else
# If this test raised an exception that means the test runner itself had some problem,
# so we may have hit a segfault, deserialization errors or something similar.
Expand All @@ -1410,6 +1423,7 @@ function _runtests(mod::Module, args::ParsedArgs;
@assert result isa Exception
testset = create_testset(testname; start, stop)
Test.record(testset, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack(NamedTuple[(;exception = result, backtrace = [])]), LineNumberNode(1)))
historical_durations[testname] = TestHistoryEntry(Inf, true)
end

with_testset(testset) do
Expand Down
37 changes: 35 additions & 2 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ end
ParallelTestRunner, parse_args(["--jobs=1", "--verbose"]);
testsuite,
tests,
historical_durations=Dict{String, Float64}(),
historical_durations=Dict{String, ParallelTestRunner.TestHistoryEntry}(),
stdout=io,
stderr=io,
)
Expand Down Expand Up @@ -820,6 +820,39 @@ end
@test any(contains("--color=yes"), exe.exec)
end

@testset "TestHistoryEntry" begin
using ParallelTestRunner: TestHistoryEntry

flow = TestHistoryEntry(1,true)
fhigh = TestHistoryEntry(10,true)

slow = TestHistoryEntry(1,false)
shigh = TestHistoryEntry(10,false)

# irreflexive: lt(x, x) always yields false
@test !isless(flow, flow)
@test !isless(slow, slow)
@test !isless(fhigh, fhigh)
@test !isless(shigh, shigh)

# asymmetric: if lt(x, y) yields true then lt(y, x) yields false
@test isless(flow, fhigh)
@test !isless(fhigh, flow)
@test isless(shigh, flow)
@test !isless(flow, shigh)

# transitive: lt(x, y) && lt(y, z) implies lt(x, z)
@test isless(slow, shigh)
@test isless(shigh, flow)
@test isless(slow, flow)

# addition/subtraction with Float64
@test TestHistoryEntry(1, true) + 1.0 == 2.0
@test TestHistoryEntry(3, false) - 1.0 == 2.0
@test TestHistoryEntry(1, false) + 1.0 == 2.0
@test TestHistoryEntry(3, true) - 1.0 == 2.0
end

# ── Integration tests ────────────────────────────────────────────────────────

@testset "non-verbose mode" begin
Expand Down Expand Up @@ -854,7 +887,7 @@ end
ParallelTestRunner, parse_args(["--quickfail", "--verbose", "--jobs=1"]);
testsuite,
tests=["fail-test", "pass-test1", "pass-test2", "pass-test3", "pass-test4", "pass-test5"],
historical_durations=Dict{String, Float64}(),
historical_durations=Dict{String, ParallelTestRunner.TestHistoryEntry}(),
stdout=io,
stderr=io,
)
Expand Down