Add retry mechanism on saving tasks to history collection

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	code.vereign.com/gaiax/tsa/golib v0.0.0-20220321093827-5fdf8f34aad9
+	github.com/cenkalti/backoff/v4 v4.1.3
 	github.com/google/uuid v1.3.0
 	github.com/kelseyhightower/envconfig v1.4.0
 	go.mongodb.org/mongo-driver v1.8.4

go.sum

@@ -49,6 +49,8 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
+github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
+github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=

internal/executor/worker.go

@@ -56,7 +56,6 @@ func (w *Worker) Start(ctx context.Context) {
 			}
 
 			if err := w.storage.SaveTaskHistory(ctx, executed); err != nil {
-				// TODO: is this fatal error, should the task be re-executed?
 				logger.Error("error saving task history", zap.Error(err))
 				continue
 			}

internal/storage/storage.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"strings"
 
+	"github.com/cenkalti/backoff/v4"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
@@ -109,6 +110,11 @@ func (s *Storage) Unack(ctx context.Context, t *task.Task) error {
 
 // SaveTaskHistory saves a task to the `tasksHistory` collection.
 func (s *Storage) SaveTaskHistory(ctx context.Context, task *task.Task) error {
-	_, err := s.tasksHistory.InsertOne(ctx, task)
-	return err
+	insert := func() error {
+		_, err := s.tasksHistory.InsertOne(ctx, task)
+		return err
+	}
+
+	b := backoff.WithContext(backoff.NewExponentialBackOff(), ctx)
+	return backoff.Retry(insert, b)
 }