diff --git a/test/integration/async_processes_lifecycle_test.rb b/test/integration/async_processes_lifecycle_test.rb index 7f2e7007..65a17ad6 100644 --- a/test/integration/async_processes_lifecycle_test.rb +++ b/test/integration/async_processes_lifecycle_test.rb @@ -128,7 +128,7 @@ class AsyncProcessesLifecycleTest < ActiveSupport::TestCase test "term supervisor exceeding timeout while there are jobs in-flight" do no_pause = enqueue_store_result_job("no pause") - pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 10.second) + pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 30.seconds) # Wait for the "no pause" job to complete and the pause job to start. # A claimed execution alone is not enough here because the worker may have @@ -153,8 +153,15 @@ class AsyncProcessesLifecycleTest < ActiveSupport::TestCase assert_completed_job_results("no pause") assert_job_status(no_pause, :finished) - # The pause job should not have completed - assert_not_equal "completed", skip_active_record_query_cache { JobResult.find_by(value: "pause")&.status } + # The pause job should not have completed. Its pause is far longer than this + # test's entire timeline, so a completed result can only come from outside + # the test's own flow — include enough state to tell where it came from. + skip_active_record_query_cache do + assert_not_equal "completed", JobResult.find_by(value: "pause")&.status, + "Expected the pause job not to complete. " \ + "Job results: #{JobResult.all.map { |r| { id: r.id, queue: r.queue_name, status: r.status, value: r.value, updated_at: r.updated_at } }.inspect}; " \ + "registered processes: #{SolidQueue::Process.all.map { |p| { id: p.id, kind: p.kind, pid: p.pid, last_heartbeat_at: p.last_heartbeat_at } }.inspect}" + end # After shutdown, the pause job may be either: # - claimed (exit! called, no cleanup) OR diff --git a/test/integration/concurrency_controls_test.rb b/test/integration/concurrency_controls_test.rb index 9cdb3d65..0f20034c 100644 --- a/test/integration/concurrency_controls_test.rb +++ b/test/integration/concurrency_controls_test.rb @@ -174,7 +174,7 @@ class ConcurrencyControlsTest < ActiveSupport::TestCase end test "don't block claimed executions that get released" do - NonOverlappingUpdateResultJob.perform_later(@result, name: "I'll be released to ready", pause: SolidQueue.shutdown_timeout + 10.seconds) + NonOverlappingUpdateResultJob.perform_later(@result, name: "I'll be released to ready", pause: SolidQueue.shutdown_timeout + 30.seconds) job = SolidQueue::Job.last wait_for(timeout: 2.seconds) { job.reload.claimed? } diff --git a/test/integration/forked_processes_lifecycle_test.rb b/test/integration/forked_processes_lifecycle_test.rb index deb196d3..036ba239 100644 --- a/test/integration/forked_processes_lifecycle_test.rb +++ b/test/integration/forked_processes_lifecycle_test.rb @@ -124,7 +124,7 @@ class ForkedProcessesLifecycleTest < ActiveSupport::TestCase test "term supervisor exceeding timeout while there are jobs in-flight" do no_pause = enqueue_store_result_job("no pause") - pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 10.seconds) + pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 30.seconds) wait_while_with_timeout(5.seconds) { SolidQueue::ReadyExecution.joins(:job).exists?(solid_queue_jobs: { active_job_id: pause.job_id }) @@ -238,12 +238,15 @@ class ForkedProcessesLifecycleTest < ActiveSupport::TestCase end test "kill worker individually" do - killed_pause = enqueue_store_result_job("killed_pause", pause: 2.seconds) + killed_pause = enqueue_store_result_job("killed_pause", pause: 5.seconds) enqueue_store_result_job("pause", :default, pause: 0.5.seconds) wait_for_jobs_to_finish_for(1.second, except: [ killed_pause ]) - # Ensure the long job has written its "started" row before we SIGKILL the worker. - wait_while_with_timeout(2.seconds) do + # Ensure the long job has written its "started" row before we SIGKILL the + # worker. Raise if it never does: proceeding would kill the worker before + # it starts the job, so no "started" row would ever exist and the test + # would fail later, in a way much harder to trace back here. + wait_while_with_timeout!(10.seconds) do JobResult.where(status: "started", value: "killed_pause").none? end diff --git a/test/unit/process_recovery_test.rb b/test/unit/process_recovery_test.rb index 296d6b95..7de69cf6 100644 --- a/test/unit/process_recovery_test.rb +++ b/test/unit/process_recovery_test.rb @@ -18,7 +18,7 @@ class ProcessRecoveryTest < ActiveSupport::TestCase test "supervisor handles missing process record and fails claimed executions properly" do # Start a supervisor with one worker @pid = run_supervisor_as_fork(workers: [ { queues: "*", polling_interval: 0.1, processes: 1 } ]) - wait_for_registered_processes(2, timeout: 1.second) # Supervisor + 1 worker + wait_for_registered_processes(2, timeout: 3.seconds) # Supervisor + 1 worker supervisor_process = SolidQueue::Process.find_by(kind: "Supervisor(fork)", pid: @pid) assert supervisor_process @@ -43,8 +43,10 @@ class ProcessRecoveryTest < ActiveSupport::TestCase worker_pid = worker_process.pid terminate_process(worker_pid, signal: :KILL) - # Wait for the supervisor to reap the worker and fail the job - wait_while_with_timeout(3.seconds) { SolidQueue::FailedExecution.none? } + # Wait for the supervisor to reap the worker and fail the job. The + # supervisor only checks for terminated forks about once a second, so give + # it enough margin for a couple of cycles even on a slow runner. + wait_while_with_timeout(10.seconds) { SolidQueue::FailedExecution.none? } # Assert the execution is failed failed_execution = SolidQueue::FailedExecution.last