Add backoff retry for store setup (#4964)

cmd/server/flags.go

@@ -56,6 +56,12 @@ var flags = append([]cli.Flag{
 		Usage:   "time an active connection is allowed to stay open",
 		Value:   3 * time.Second,
 	},
+	&cli.UintFlag{
+		Sources: cli.EnvVars("WOODPECKER_DATABASE_MAX_RETRIES"),
+		Name:    "db-max-retries",
+		Usage:   "max number of retries for the initial connection to the database",
+		Value:   10,
+	},
 	&cli.StringFlag{
 		Sources: cli.EnvVars("WOODPECKER_HOST"),
 		Name:    "server-host",

cmd/server/server.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v5"
 	"github.com/gin-gonic/gin"
 	prometheus_http "github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
@@ -36,6 +37,7 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v3/server/cron"
 	"go.woodpecker-ci.org/woodpecker/v3/server/router"
 	"go.woodpecker-ci.org/woodpecker/v3/server/router/middleware"
+	"go.woodpecker-ci.org/woodpecker/v3/server/store"
 	"go.woodpecker-ci.org/woodpecker/v3/server/web"
 	"go.woodpecker-ci.org/woodpecker/v3/shared/logger"
 	"go.woodpecker-ci.org/woodpecker/v3/version"
@@ -91,10 +93,19 @@ func run(ctx context.Context, c *cli.Command) error {
 		)
 	}
 
-	_store, err := setupStore(ctx, c)
+	_store, err := backoff.Retry(ctx,
+		func() (store.Store, error) {
+			return setupStore(ctx, c)
+		},
+		backoff.WithBackOff(backoff.NewExponentialBackOff()),
+		backoff.WithMaxTries(uint(c.Uint("db-max-retries"))),
+		backoff.WithNotify(func(err error, delay time.Duration) {
+			log.Error().Msgf("failed to setup store: %v: retry in %v", err, delay)
+		}))
 	if err != nil {
-		return fmt.Errorf("can't setup store: %w", err)
+		return err
 	}
+
 	defer func() {
 		if err := _store.Close(); err != nil {
 			log.Error().Err(err).Msg("could not close store")

cmd/server/setup.go

@@ -90,6 +90,10 @@ func setupStore(ctx context.Context, c *cli.Command) (store.Store, error) {
 		return nil, fmt.Errorf("could not open datastore: %w", err)
 	}
 
+	if err = store.Ping(); err != nil {
+		return nil, err
+	}
+
 	if err := store.Migrate(ctx, c.Bool("migrations-allow-long")); err != nil {
 		return nil, fmt.Errorf("could not migrate datastore: %w", err)
 	}