From debcbe3c1e31ed200689cf7ccfd27287d7b57755 Mon Sep 17 00:00:00 2001 From: Chris George Date: Sun, 23 Aug 2026 14:32:18 -0700 Subject: [PATCH 1/2] Reconcile active jobs against cancel_attempted_at after notifier reconnects --- CHANGELOG.md | 1 + internal/notifier/notifier.go | 59 ++++++++-- internal/notifier/notifier_test.go | 48 ++++++++ producer.go | 119 ++++++++++++++++++- producer_test.go | 178 +++++++++++++++++++++++++++++ 5 files changed, 394 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 263d126d..d2de904f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed periodic jobs advancing their durable next run time when job insertion fails. [PR #1359](https://github.com/riverqueue/river/pull/1359). +- A `JobCancel()` sent while a client's notifier is disconnected/reconnecting is no longer lost. Previously, the cancellation's `NOTIFY` could commit while nothing was `LISTEN`ing (Postgres `NOTIFY` is fire-and-forget), leaving the running job unaware of the cancellation until `JobRescuer`'s much longer stuck-job sweep. Each client now reconciles its currently running jobs against `cancel_attempted_at` by primary key every time its notifier (re)establishes healthy listening, closing the gap with no added steady-state query load. [PR #1361](https://github.com/riverqueue/river/pull/1361), as discussed in [#1358](https://github.com/riverqueue/river/issues/1358). ## [0.44.1] - 2026-08-21 diff --git a/internal/notifier/notifier.go b/internal/notifier/notifier.go index 8b48fbdd..f556fca3 100644 --- a/internal/notifier/notifier.go +++ b/internal/notifier/notifier.go @@ -88,12 +88,46 @@ type Notifier struct { testSignals notifierTestSignals waitInterruptChan chan func() - mu sync.RWMutex - isConnected bool - isStarted bool - isWaiting bool - subscriptions map[NotificationTopic][]*Subscription - waitCancel context.CancelFunc + mu sync.RWMutex + isConnected bool + isStarted bool + isWaiting bool + listenerReadyFuncs map[*ListenerReadySubscription]ListenerReadyFunc + subscriptions map[NotificationTopic][]*Subscription + waitCancel context.CancelFunc +} + +// ListenerReadyFunc is invoked each time the notifier establishes healthy +// listening on all of its currently subscribed topics. This includes the +// very first connect as well as every subsequent reconnect that follows a +// connection loss, so callers whose work is only meaningful after a genuine +// reconnect (as opposed to the initial connect) should make that check +// themselves. +type ListenerReadyFunc func() + +// ListenerReadySubscription is a handle to a func registered with +// RegisterListenerReadyFunc. Call Unregister to stop receiving invocations. +type ListenerReadySubscription struct { + notifier *Notifier +} + +func (s *ListenerReadySubscription) Unregister() { + s.notifier.mu.Lock() + defer s.notifier.mu.Unlock() + delete(s.notifier.listenerReadyFuncs, s) +} + +// RegisterListenerReadyFunc registers a function to be invoked every time the +// notifier (re)establishes healthy listening on all of its subscribed topics. +// It's used to let callers reconcile any state that could have missed a +// notification while the notifier was disconnected/reconnecting. +func (n *Notifier) RegisterListenerReadyFunc(listenerReadyFunc ListenerReadyFunc) *ListenerReadySubscription { + n.mu.Lock() + defer n.mu.Unlock() + + sub := &ListenerReadySubscription{notifier: n} + n.listenerReadyFuncs[sub] = listenerReadyFunc + return sub } func New(archetype *baseservice.Archetype, listener riverdriver.Listener) *Notifier { @@ -102,7 +136,8 @@ func New(archetype *baseservice.Archetype, listener riverdriver.Listener) *Notif notificationBuf: make(chan *riverdriver.Notification, 1000), waitInterruptChan: make(chan func(), 10), - subscriptions: make(map[NotificationTopic][]*Subscription), + listenerReadyFuncs: make(map[*ListenerReadySubscription]ListenerReadyFunc), + subscriptions: make(map[NotificationTopic][]*Subscription), }) return notifier } @@ -212,6 +247,16 @@ func (n *Notifier) listenAndWait(ctx context.Context) error { n.testSignals.ListeningBegin.Signal(struct{}{}) defer n.testSignals.ListeningEnd.Signal(struct{}{}) + listenerReadyFuncs := func() []ListenerReadyFunc { + n.mu.RLock() + defer n.mu.RUnlock() + + return maputil.Values(n.listenerReadyFuncs) + }() + for _, listenerReadyFunc := range listenerReadyFuncs { + listenerReadyFunc() + } + drainInterrupts := func() { for { select { diff --git a/internal/notifier/notifier_test.go b/internal/notifier/notifier_test.go index a6dc16ab..37d3bd7e 100644 --- a/internal/notifier/notifier_test.go +++ b/internal/notifier/notifier_test.go @@ -622,6 +622,54 @@ func TestNotifier(t *testing.T) { // Subscription should still work. require.Equal(t, TopicAndPayload{testTopic1, "msg1"}, riversharedtest.WaitOrTimeout(t, notifyChan)) }) + + t.Run("RegisterListenerReadyFunc", func(t *testing.T) { + t.Parallel() + + notifier, _ := setup(t, nil) + notifier.testDisableSleep = true + + var errorNum int + + listenerMock := NewListenerMock(notifier.listener) + listenerMock.waitForNotificationFunc = func(ctx context.Context) (*riverdriver.Notification, error) { + // Returns an error the first time, but then works after. + errorNum++ + switch errorNum { + case 1: + return nil, errors.New("error during wait") + default: + return listenerMock.Listener.WaitForNotification(ctx) + } + } + notifier.listener = listenerMock + + readyChan := make(chan struct{}, 10) + sub := notifier.RegisterListenerReadyFunc(func() { readyChan <- struct{}{} }) + t.Cleanup(sub.Unregister) + + start(t, notifier) + + // Fires once on the initial connect. + riversharedtest.WaitOrTimeout(t, readyChan) + + // And again once the notifier reconnects after the injected failure. + require.EqualError(t, notifier.testSignals.BackoffError.WaitOrTimeout(), "error during wait") + riversharedtest.WaitOrTimeout(t, readyChan) + + // Unregister stops further invocations. There's no direct signal for + // "would have fired again", so drain what's pending, unregister, and + // confirm a subsequent restart doesn't add anything new. + for len(readyChan) > 0 { + <-readyChan + } + sub.Unregister() + + notifier.Stop() + require.NoError(t, notifier.Start(ctx)) + notifier.testSignals.ListeningBegin.WaitOrTimeout() + require.Empty(t, readyChan) + }) } type ListenerMock struct { diff --git a/producer.go b/producer.go index da11246e..94d4d235 100644 --- a/producer.go +++ b/producer.go @@ -229,8 +229,22 @@ type producer struct { // Receives control messages from the notifier goroutine. Written by notifier // goroutine, only read from main goroutine. queueControlCh chan *controlEventPayload - retryPolicy ClientRetryPolicy - testSignals producerTestSignals + + // Receives a signal every time the notifier (re)establishes healthy + // listening. Written by the notifier goroutine (via a registered + // ListenerReadyFunc), only read from main goroutine. Used to reconcile + // activeJobs against cancel_attempted_at in case a cancel NOTIFY was lost + // while the notifier was disconnected/reconnecting. Buffered to 1 and + // written to non-blockingly because reconnects are rare and coalescing + // multiple signals into one reconciliation pass is fine. + reconnectCh chan struct{} + + // Only used by main goroutine. Tracks the current backoff attempt for + // retrying the reconnect reconciliation query after a failure. + reconcileAttempt int + + retryPolicy ClientRetryPolicy + testSignals producerTestSignals } func newProducer(archetype *baseservice.Archetype, exec riverdriver.Executor, pilot riverpilot.Pilot, config *producerConfig) *producer { @@ -257,6 +271,7 @@ func newProducer(archetype *baseservice.Archetype, exec riverdriver.Executor, pi jobTimeout: config.JobTimeout, pilot: pilot, queueControlCh: make(chan *controlEventPayload, 100), + reconnectCh: make(chan struct{}, 1), retryPolicy: config.RetryPolicy, workers: config.Workers, }) @@ -351,8 +366,9 @@ func (p *producer) StartWorkContext(fetchCtx, workCtx context.Context) error { p.fetchLimiter = chanutil.NewDebouncedChan(fetchCtx, p.config.FetchCooldown, true) var ( - controlSub *notifier.Subscription - insertSub *notifier.Subscription + controlSub *notifier.Subscription + insertSub *notifier.Subscription + reconnectSub *notifier.ListenerReadySubscription ) if p.config.Notifier != nil { var err error @@ -386,6 +402,21 @@ func (p *producer) StartWorkContext(fetchCtx, workCtx context.Context) error { } return err } + + // Reconcile activeJobs against cancel_attempted_at every time the + // notifier (re)establishes healthy listening. This closes the window + // described in https://github.com/riverqueue/river/issues/1358: a + // cancel NOTIFY sent while the notifier was disconnected/reconnecting + // is otherwise lost for good (Postgres NOTIFY is fire-and-forget). + // Firing on the very first connect too is harmless: activeJobs is + // always empty at that point in startup, so the fetchAndRunLoop + // handler below is a no-op query-free check. + reconnectSub = p.config.Notifier.RegisterListenerReadyFunc(func() { + select { + case p.reconnectCh <- struct{}{}: + default: + } + }) } go func() { @@ -405,6 +436,10 @@ func (p *producer) StartWorkContext(fetchCtx, workCtx context.Context) error { defer controlSub.Unlisten(fetchCtx) } + if reconnectSub != nil { + defer reconnectSub.Unregister() + } + var subroutineWG sync.WaitGroup subroutineCtx, cancelSubroutines := context.WithCancelCause(context.WithoutCancel(fetchCtx)) @@ -581,6 +616,8 @@ func (p *producer) fetchAndRunLoop(fetchCtx, workCtx context.Context) { } case jobID := <-p.cancelCh: p.maybeCancelJob(workCtx, jobID) + case <-p.reconnectCh: + p.handleNotifierReconnect(workCtx) case <-p.fetchLimiter.C(): p.innerFetchLoop(workCtx, fetchResultCh) // Ensure we can't start another fetch when fetchCtx is done, even if @@ -788,6 +825,80 @@ func (p *producer) maybeCancelJob(ctx context.Context, id int64) { executor.Cancel(ctx) } +// jobMetadataWithCancelAttemptedAt mirrors the identical unexported struct in +// internal/maintenance/job_rescuer.go, which reads the same field for the +// same reason. +type jobMetadataWithCancelAttemptedAt struct { + CancelAttemptedAt time.Time `json:"cancel_attempted_at"` +} + +// handleNotifierReconnect reconciles activeJobs against cancel_attempted_at +// after the notifier has (re)established healthy listening. It exists to +// close the window described in +// https://github.com/riverqueue/river/issues/1358: a cancel NOTIFY sent while +// the notifier was disconnected/reconnecting is lost for good (Postgres +// NOTIFY is fire-and-forget), so a job cancelled during that window would +// otherwise run to completion unaware, undetected until JobRescuer's much +// longer stuck-job sweep. +// +// Only ever called from the main goroutine, so activeJobs can be read +// without a lock. +func (p *producer) handleNotifierReconnect(ctx context.Context) { + if len(p.activeJobs) == 0 { + p.reconcileAttempt = 0 + return + } + + ids := make([]int64, 0, len(p.activeJobs)) + for id := range p.activeJobs { + ids = append(ids, id) + } + + jobs, err := p.exec.JobGetByIDMany(ctx, &riverdriver.JobGetByIDManyParams{ + ID: ids, + Schema: p.config.Schema, + }) + if err != nil { + p.Logger.ErrorContext(ctx, p.Name+": Error reconciling active jobs against cancel_attempted_at after notifier reconnect; will retry", + slog.String("err", err.Error())) + p.scheduleReconnectRetry(ctx) + return + } + p.reconcileAttempt = 0 + + for _, job := range jobs { + var metadata jobMetadataWithCancelAttemptedAt + if err := json.Unmarshal(job.Metadata, &metadata); err != nil { + p.Logger.ErrorContext(ctx, p.Name+": Error unmarshaling job metadata while reconciling cancellations after notifier reconnect", + slog.Int64("job_id", job.ID), slog.String("err", err.Error())) + continue + } + if !metadata.CancelAttemptedAt.IsZero() { + p.maybeCancelJob(ctx, job.ID) + } + } +} + +// scheduleReconnectRetry re-arms p.reconnectCh after a backoff so a +// reconciliation query that failed (e.g. a transient DB error) doesn't +// silently drop the only signal that a reconnect happened, which would +// recreate the lost-cancel window this mechanism exists to close. +func (p *producer) scheduleReconnectRetry(ctx context.Context) { + if ctx.Err() != nil { + return + } + + sleepDuration := serviceutil.ExponentialBackoff(p.reconcileAttempt, serviceutil.MaxAttemptsBeforeResetDefault) + p.reconcileAttempt++ + + time.AfterFunc(sleepDuration, func() { + select { + case p.reconnectCh <- struct{}{}: + default: + } + }) +} + func (p *producer) metricEmitHooksFromLookup() []rivertype.HookMetricEmit { pluginLookup := p.config.PluginLookupGlobal if pluginLookup == nil { diff --git a/producer_test.go b/producer_test.go index a7041433..e8f29d72 100644 --- a/producer_test.go +++ b/producer_test.go @@ -3,8 +3,10 @@ package river import ( "context" "encoding/json" + "errors" "fmt" "slices" + "sync" "sync/atomic" "testing" "time" @@ -53,6 +55,182 @@ func (p *beforeJobGetAvailablePilot) JobGetAvailable( return p.Pilot.JobGetAvailable(ctx, exec, state, params) } +// producerTestListenerMock wraps a real riverdriver.Listener so tests can +// override individual operations. Unlike notifier.ListenerMock (unexported to +// the internal/notifier package), this lives in package river so it can be +// used to drive a real *notifier.Notifier from producer-level tests. +type producerTestListenerMock struct { + riverdriver.Listener + + connectFunc func(ctx context.Context) error + waitForNotificationFunc func(ctx context.Context) (*riverdriver.Notification, error) +} + +func newProducerTestListenerMock(listener riverdriver.Listener) *producerTestListenerMock { + return &producerTestListenerMock{ + Listener: listener, + connectFunc: listener.Connect, + waitForNotificationFunc: listener.WaitForNotification, + } +} + +func (l *producerTestListenerMock) Connect(ctx context.Context) error { return l.connectFunc(ctx) } + +func (l *producerTestListenerMock) WaitForNotification(ctx context.Context) (*riverdriver.Notification, error) { + return l.waitForNotificationFunc(ctx) +} + +// TestProducer_JobCancelSurvivesNotifierReconnect is a deterministic repro of +// https://github.com/riverqueue/river/issues/1358: a `NOTIFY` sent while the +// notifier is disconnected/reconnecting is lost (Postgres `NOTIFY` is +// fire-and-forget), so a job cancelled during that window previously ran to +// completion unaware. It forces exactly one simulated connection loss between +// the job starting and `JobCancel` being applied, holds the notifier at the +// start of its reconnect attempt until the cancellation has landed, then lets +// it finish reconnecting and asserts the job observes the cancellation. +func TestProducer_JobCancelSurvivesNotifierReconnect(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + var ( + archetype = riversharedtest.BaseServiceArchetype(t) + dbPool = riversharedtest.DBPool(ctx, t) + driver = riverpgxv5.New(dbPool) + exec = driver.GetExecutor() + jobUpdates = make(chan []jobcompleter.CompleterJobUpdated, 10) + schema = riverdbtest.TestSchema(ctx, t, driver, nil) + queueName = fmt.Sprintf("test-reconnect-cancel-%05d", randutil.IntBetween(1, 100_000)) + pilot = &riverpilot.StandardPilot{} + ) + + completer := jobcompleter.NewInlineCompleter(archetype, schema, exec, pilot, jobUpdates) + require.NoError(t, completer.Start(ctx)) + t.Cleanup(completer.Stop) + + realListener := driver.GetListener(&riverdriver.GetListenenerParams{Schema: schema}) + mockListener := newProducerTestListenerMock(realListener) + + var ( + forceDisconnect atomic.Bool // set by the test to inject one simulated connection loss + reconnecting atomic.Bool // set once that loss has been injected; cleared once Connect is released + releaseReconnect = make(chan struct{}) + releaseReconnectOnce sync.Once + ) + closeReleaseReconnect := func() { releaseReconnectOnce.Do(func() { close(releaseReconnect) }) } + mockListener.waitForNotificationFunc = func(ctx context.Context) (*riverdriver.Notification, error) { + if forceDisconnect.CompareAndSwap(true, false) { + reconnecting.Store(true) + return nil, errors.New("simulated connection loss") + } + return realListener.WaitForNotification(ctx) + } + mockListener.connectFunc = func(ctx context.Context) error { + if reconnecting.Load() { + <-releaseReconnect + reconnecting.Store(false) + } + return realListener.Connect(ctx) + } + + notif := notifier.New(archetype, mockListener) + require.NoError(t, notif.Start(ctx)) + t.Cleanup(notif.Stop) + // Cleanups run LIFO, so registering this after notif.Stop makes it run + // first: an assertion failure anywhere in the test body must not leave + // notif.Stop() deadlocked waiting on a mocked Connect call that's + // holding the reconnect gate open forever. + t.Cleanup(closeReleaseReconnect) + + jobStarted := make(chan int64, 1) + workers := NewWorkers() + + type JobArgs struct { + testutil.JobArgsReflectKind[JobArgs] + } + + AddWorker(workers, WorkFunc(func(ctx context.Context, job *Job[JobArgs]) error { + jobStarted <- job.ID + <-ctx.Done() + return ctx.Err() + })) + + prod := newProducer(archetype, exec, pilot, &producerConfig{ + ClientID: testClientID, + Completer: completer, + ErrorHandler: newTestErrorHandler(), + FetchCooldown: FetchCooldownDefault, + FetchPollInterval: 50 * time.Millisecond, + PluginLookupByJob: pluginlookup.NewJobPluginLookup(nil), + PluginLookupGlobal: pluginlookup.NewPluginLookup(nil), + JobTimeout: JobTimeoutDefault, + MaxWorkers: 10, + Notifier: notif, + Queue: queueName, + QueuePollInterval: queuePollIntervalDefault, + QueueReportInterval: queueReportIntervalDefault, + RetryPolicy: &DefaultClientRetryPolicy{}, + SchedulerInterval: riverinternaltest.SchedulerShortInterval, + Schema: schema, + StaleProducerRetentionPeriod: time.Minute, + Workers: workers, + }) + + config := newTestConfig(t, schema) + insertParams, err := insertParamsFromConfigArgsAndOptions(&prod.Archetype, config, &JobArgs{}, &InsertOpts{Queue: queueName}) + require.NoError(t, err) + _, err = exec.JobInsertFastMany(ctx, &riverdriver.JobInsertFastManyParams{ + Jobs: []*riverdriver.JobInsertFastParams{(*riverdriver.JobInsertFastParams)(insertParams)}, + Schema: schema, + }) + require.NoError(t, err) + + workCtx, workCancel := context.WithCancel(ctx) + + require.NoError(t, prod.StartWorkContext(ctx, workCtx)) + // Cleanups run LIFO: cancel workCtx (unblocking the worker) before + // prod.Stop tries to wait for it to exit, so cleanup can't deadlock + // regardless of whether the fix under test actually cancelled the job + // itself. + t.Cleanup(prod.Stop) + t.Cleanup(workCancel) + + jobID := riversharedtest.WaitOrTimeout(t, jobStarted) + + // Arm the injected disconnect, then wake the notifier's currently-blocked + // WaitForNotification call with a harmless notification on an unrelated + // queue so it loops back around and picks up the injected error + // immediately (rather than waiting on the notifier's own ping interval). + forceDisconnect.Store(true) + require.NoError(t, exec.NotifyMany(ctx, &riverdriver.NotifyManyParams{ + Topic: string(notifier.NotificationTopicInsert), + Payload: []string{`{"queue":"` + queueName + `-unrelated-wakeup"}`}, + Schema: schema, + })) + + // Wait until the notifier has actually observed the injected error and is + // blocked trying to reconnect. + require.Eventually(t, reconnecting.Load, 5*time.Second, 5*time.Millisecond, + "notifier never reached its reconnect attempt after the injected connection loss") + + // While the notifier is down, cancel the running job. The NOTIFY this + // sends is lost because nothing is LISTENing right now. + _, err = exec.JobCancel(ctx, &riverdriver.JobCancelParams{ + ID: jobID, + CancelAttemptedAt: time.Now().UTC(), + ControlTopic: string(notifier.NotificationTopicControl), + Schema: schema, + }) + require.NoError(t, err) + + // Now let the notifier finish reconnecting. + closeReleaseReconnect() + + update := riversharedtest.WaitOrTimeout(t, jobUpdates) + require.Equal(t, rivertype.JobStateCancelled, update[0].Job.State, + "job should have observed cancellation after the notifier reconnected and reconciled active jobs against cancel_attempted_at") +} + func TestProducer_MetricEmitHook(t *testing.T) { t.Parallel() From 10c23cfca1444624bd273597e385a151aee940df Mon Sep 17 00:00:00 2001 From: Chris George Date: Sun, 23 Aug 2026 14:53:09 -0700 Subject: [PATCH 2/2] Deliver cancellation locally when the client has no notifier --- CHANGELOG.md | 1 + client.go | 73 +++++++++++++++++++++++++++--- client_test.go | 118 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2de904f..6a1fa600 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed periodic jobs advancing their durable next run time when job insertion fails. [PR #1359](https://github.com/riverqueue/river/pull/1359). - A `JobCancel()` sent while a client's notifier is disconnected/reconnecting is no longer lost. Previously, the cancellation's `NOTIFY` could commit while nothing was `LISTEN`ing (Postgres `NOTIFY` is fire-and-forget), leaving the running job unaware of the cancellation until `JobRescuer`'s much longer stuck-job sweep. Each client now reconciles its currently running jobs against `cancel_attempted_at` by primary key every time its notifier (re)establishes healthy listening, closing the gap with no added steady-state query load. [PR #1361](https://github.com/riverqueue/river/pull/1361), as discussed in [#1358](https://github.com/riverqueue/river/issues/1358). +- `JobCancel()` against a job running in the same process is now delivered immediately for clients with no notifier at all, including an explicit `Config.PollOnly` on an otherwise listener-capable driver. Previously, the local delivery path only activated for drivers that can't support `LISTEN`/`NOTIFY` at all, so `PollOnly: true` on the Postgres or SQLite drivers fell all the way back to `JobRescuer`'s stuck-job sweep for a job it could have cancelled synchronously in-process. [PR #1361](https://github.com/riverqueue/river/pull/1361). ## [0.44.1] - 2026-08-21 diff --git a/client.go b/client.go index bf07bfc0..5ea95b79 100644 --- a/client.go +++ b/client.go @@ -1456,11 +1456,22 @@ func (c *Client[TTx]) JobCancel(ctx context.Context, jobID int64) (*rivertype.Jo return nil, err } - c.notifyProducerWithoutListenerQueueControlEvent(job.Queue, &controlEventPayload{ - Action: controlActionCancel, - JobID: job.ID, - Queue: job.Queue, - }) + // Only a running job has an in-process executor to interrupt (an + // available/scheduled/retryable job was cancelled directly above, and a + // finalized job needed no cancellation at all), so there's nothing for a + // non-running job to dispatch. This also matters for correctness, not + // just efficiency: it keeps a burst of cancellations against jobs that + // were never running (e.g. cancelling many freshly-inserted jobs before + // the client has even been started) from filling the producer's + // control-event channel, which TriggerQueueControlEvent below sends to + // unconditionally and would otherwise block this call indefinitely. + if job.State == rivertype.JobStateRunning { + c.notifyProducerOfCancelWithoutNotifier(job.Queue, &controlEventPayload{ + Action: controlActionCancel, + JobID: job.ID, + Queue: job.Queue, + }) + } return job, nil } @@ -2876,6 +2887,58 @@ func (c *Client[TTx]) notifyProducerWithoutListenerQueueControlEvent(queue strin } } +// Notifies an internal producer of a job cancellation so a job running in +// this same process observes it immediately, for clients that have no +// notifier to deliver it via listen/notify. +// +// This deliberately does NOT share notifyProducerWithoutListenerQueueControlEvent +// above despite the near-identical dispatch: the two helpers guard on +// different conditions for a reason, and unifying them would either reopen +// this gap or introduce new bugs in the other one. +// +// - This helper guards on c.notifier == nil (this client has no notifier, +// for any reason, including an explicit Config.PollOnly on an otherwise +// listener-capable driver). Queue control events guard on +// driver.SupportsListener() instead, because pause/resume/metadata +// changes already have an independent, documented convergence path in +// poll-only mode (producer.pollForSettingChanges, on QueuePollInterval) — +// using this same client.notifier == nil guard there would double-deliver +// metadata changes not deduplicated the way pause/resume are, and would +// panic on QueueUpdate's nil control event when metadata is empty. +// - Job cancellation has no such fallback: nothing else ever re-checks a +// running job's cancel_attempted_at in poll-only mode, so a lost signal +// here is only caught by JobRescuer's much longer stuck-job sweep. +// +// The dispatch below (via TriggerQueueControlEvent) sends unconditionally and +// blocks if the target producer's control-event channel is full — the same +// characteristic notifyProducerWithoutListenerQueueControlEvent above already +// has for genuinely listener-incapable drivers. The caller (JobCancel) is +// responsible for only invoking this for jobs it knows are actually running, +// both because there's nothing to dispatch for a job with no in-process +// executor and to avoid a burst of cancellations against non-running jobs +// (e.g. many freshly-inserted jobs cancelled before the client is started) +// filling that channel and blocking indefinitely. +// +// Should only ever be invoked *outside* a transaction. If invoked within a +// transaction, the producer wouldn't yet be able to access the state that +// triggered the notification because it's not committed yet. +func (c *Client[TTx]) notifyProducerOfCancelWithoutNotifier(queue string, controlEvent *controlEventPayload) { + if c.notifier != nil { + return + } + + c.producersMu.RLock() + defer c.producersMu.RUnlock() + + if len(c.producersByQueueName) < 1 { + return + } + + if producer, ok := c.producersByQueueName[queue]; ok { + producer.TriggerQueueControlEvent(controlEvent) + } +} + func (c *Client[TTx]) queueUpdate(ctx context.Context, executorTx riverdriver.ExecutorTx, name string, params *QueueUpdateParams) (*rivertype.Queue, *controlEventPayload, error) { updateMetadata := len(params.Metadata) > 0 diff --git a/client_test.go b/client_test.go index bb7ba815..1147f2b0 100644 --- a/client_test.go +++ b/client_test.go @@ -1368,6 +1368,112 @@ func Test_Client_Common(t *testing.T) { require.WithinDuration(t, time.Now(), *event.Job.FinalizedAt, 2*time.Second) }) + // Unlike CancelRunningJobPollOnly above (which uses a fake driver that + // reports SupportsListener() == false to simulate a driver that can't + // listen/notify at all), this uses the real pgx driver with an explicit + // Config.PollOnly, which is the actually-common "I don't want a notifier" + // case. driver.SupportsListener() is true here, so a same-process + // cancellation delivered via JobCancel() has nothing to do with the + // listen/notify path at all: it must go through the client's local, + // no-notifier dispatch instead. + t.Run("CancelRunningJobExplicitPollOnly", func(t *testing.T) { + t.Parallel() + + config, bundle := setupConfig(t) + config.PollOnly = true + + client, err := NewClient(bundle.driver, config) + require.NoError(t, err) + + jobStartedChan := make(chan int64) + + type JobArgs struct { + testutil.JobArgsReflectKind[JobArgs] + } + + AddWorker(client.config.Workers, WorkFunc(func(ctx context.Context, job *Job[JobArgs]) error { + jobStartedChan <- job.ID + <-ctx.Done() + return ctx.Err() + })) + + subscribeChan := subscribe(t, client) + startClient(ctx, t, client) + riversharedtest.WaitOrTimeout(t, client.baseStartStop.Started()) + + insertRes, err := client.Insert(ctx, &JobArgs{}, nil) + require.NoError(t, err) + + startedJobID := riversharedtest.WaitOrTimeout(t, jobStartedChan) + require.Equal(t, insertRes.Job.ID, startedJobID) + + updatedJob, err := client.JobCancel(ctx, insertRes.Job.ID) + require.NoError(t, err) + require.Equal(t, rivertype.JobStateRunning, updatedJob.State) + + event := riversharedtest.WaitOrTimeout(t, subscribeChan) + require.Equal(t, EventKindJobCancelled, event.Kind) + require.Equal(t, rivertype.JobStateCancelled, event.Job.State) + require.WithinDuration(t, time.Now(), *event.Job.FinalizedAt, 2*time.Second) + }) + + // A non-running job (available, scheduled, or retryable) is cancelled + // directly in the database and has no in-process executor to notify, so + // JobCancel must not dispatch a local control event for it. This matters + // beyond efficiency: notifyProducerOfCancelWithoutNotifier's dispatch + // blocks if the target producer's control-event channel (buffered to + // 100) fills up, which an unstarted (or otherwise not-yet-draining) + // producer never does. Cancelling more than that many never-started jobs + // against an explicit-PollOnly client (which has no notifier and so + // would otherwise take the local dispatch path) must not block. + t.Run("CancelManyNonRunningJobsExplicitPollOnlyDoesNotBlock", func(t *testing.T) { + t.Parallel() + + config, bundle := setupConfig(t) + config.PollOnly = true + + client, err := NewClient(bundle.driver, config) + require.NoError(t, err) + + type JobArgs struct { + testutil.JobArgsReflectKind[JobArgs] + } + + AddWorker(client.config.Workers, WorkFunc(func(ctx context.Context, job *Job[JobArgs]) error { + return nil + })) + + const numJobs = 101 // more than the producer's 100-slot control-event buffer + + jobIDs := make([]int64, numJobs) + for i := range jobIDs { + insertRes, err := client.Insert(ctx, &JobArgs{}, nil) + require.NoError(t, err) + jobIDs[i] = insertRes.Job.ID + } + + // Deliberately cancelling before the client is started: nothing is + // running yet, and nothing is consuming the (not-yet-existent) + // producer's control-event channel either. + cancelDone := make(chan error, 1) + go func() { + for _, jobID := range jobIDs { + if _, err := client.JobCancel(ctx, jobID); err != nil { + cancelDone <- err + return + } + } + cancelDone <- nil + }() + + select { + case err := <-cancelDone: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("cancelling non-running jobs blocked, presumably on the producer's control-event channel") + } + }) + t.Run("CancelScheduledJob", func(t *testing.T) { t.Parallel() @@ -1433,15 +1539,27 @@ func Test_Client_Common(t *testing.T) { testutil.JobArgsReflectKind[JobArgs] } + jobStartedChan := make(chan int64) + continueChan := make(chan struct{}) + AddWorker(client.config.Workers, WorkFunc(func(ctx context.Context, job *Job[JobArgs]) error { + jobStartedChan <- job.ID + <-continueChan return nil })) startClient(ctx, t, client) + t.Cleanup(func() { close(continueChan) }) insertRes, err := client.Insert(ctx, &JobArgs{}, nil) require.NoError(t, err) + // Only a running job dispatches a local control event (see + // notifyProducerOfCancelWithoutNotifier), so make sure the job has + // actually started before cancelling it. + startedJobID := riversharedtest.WaitOrTimeout(t, jobStartedChan) + require.Equal(t, insertRes.Job.ID, startedJobID) + _, err = client.JobCancel(ctx, insertRes.Job.ID) require.NoError(t, err)