Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions backend/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ type worker struct {
processor TaskProcessor
waiting bool
stop atomic.Bool
// consecutiveErrors tracks connection failures for reconnection logging
consecutiveErrors atomic.Int32
}

type NewTaskWorkerOptions func(*WorkerOptions)
Expand Down Expand Up @@ -113,6 +115,9 @@ func (w *worker) Start(ctx context.Context) {
switch {
case ok:
// found a work item - reset the backoff and check for the next item
if w.waiting {
w.logger.Infof("%v: reconnected and ready to process work items", w.Name())
}
b.Reset()
case err != nil && errors.Is(err, ctx.Err()):
// there's an error and it's due to the context being canceled
Expand Down Expand Up @@ -174,6 +179,11 @@ func (w *worker) ProcessNext(ctx context.Context) (bool, error) {
wi, err := w.processor.FetchWorkItem(ctx)
switch {
case errors.Is(err, ErrNoWorkItems) || wi == nil:
// Check if we recovered from errors when no work items
if w.consecutiveErrors.Load() > 0 {
w.logger.Infof("%v: reconnected and ready to process work items", w.Name())
w.consecutiveErrors.Store(0)
}
Comment thread
jeanmartins marked this conversation as resolved.
if !w.waiting {
w.logger.Debugf("%v: waiting for new work items...", w.Name())
w.waiting = true
Expand All @@ -182,9 +192,16 @@ func (w *worker) ProcessNext(ctx context.Context) (bool, error) {
case err != nil:
if !errors.Is(err, ctx.Err()) {
w.logger.Errorf("%v: failed to fetch work item: %v", w.Name(), err)
// Increment error counter for fetch failures
w.consecutiveErrors.Add(1)
}
return false, err
default:
// Check if we recovered from errors when successfully fetching work item
if w.consecutiveErrors.Load() > 0 {
w.logger.Infof("%v: reconnected and ready to process work items", w.Name())
w.consecutiveErrors.Store(0)
}
Comment thread
jeanmartins marked this conversation as resolved.
// process the work-item in the background
w.waiting = false
processing = true
Expand Down
Loading