diff --git a/cmd/server/flags.go b/cmd/server/flags.go index 2f50828d1..8a7839457 100644 --- a/cmd/server/flags.go +++ b/cmd/server/flags.go @@ -56,6 +56,12 @@ var flags = append([]cli.Flag{ Usage: "time an active connection is allowed to stay open", Value: 3 * time.Second, }, + &cli.UintFlag{ + Sources: cli.EnvVars("WOODPECKER_DATABASE_MAX_RETRIES"), + Name: "db-max-retries", + Usage: "max number of retries for the initial connection to the database", + Value: 10, + }, &cli.StringFlag{ Sources: cli.EnvVars("WOODPECKER_HOST"), Name: "server-host", diff --git a/cmd/server/server.go b/cmd/server/server.go index b6ab3c3b7..064ba8b08 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/cenkalti/backoff/v5" "github.com/gin-gonic/gin" prometheus_http "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" @@ -36,6 +37,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/server/cron" "go.woodpecker-ci.org/woodpecker/v3/server/router" "go.woodpecker-ci.org/woodpecker/v3/server/router/middleware" + "go.woodpecker-ci.org/woodpecker/v3/server/store" "go.woodpecker-ci.org/woodpecker/v3/server/web" "go.woodpecker-ci.org/woodpecker/v3/shared/logger" "go.woodpecker-ci.org/woodpecker/v3/version" @@ -91,10 +93,19 @@ func run(ctx context.Context, c *cli.Command) error { ) } - _store, err := setupStore(ctx, c) + _store, err := backoff.Retry(ctx, + func() (store.Store, error) { + return setupStore(ctx, c) + }, + backoff.WithBackOff(backoff.NewExponentialBackOff()), + backoff.WithMaxTries(uint(c.Uint("db-max-retries"))), + backoff.WithNotify(func(err error, delay time.Duration) { + log.Error().Msgf("failed to setup store: %v: retry in %v", err, delay) + })) if err != nil { - return fmt.Errorf("can't setup store: %w", err) + return err } + defer func() { if err := _store.Close(); err != nil { log.Error().Err(err).Msg("could not close store") diff --git a/cmd/server/setup.go b/cmd/server/setup.go index bb7a575ff..6481714bd 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -90,6 +90,10 @@ func setupStore(ctx context.Context, c *cli.Command) (store.Store, error) { return nil, fmt.Errorf("could not open datastore: %w", err) } + if err = store.Ping(); err != nil { + return nil, err + } + if err := store.Migrate(ctx, c.Bool("migrations-allow-long")); err != nil { return nil, fmt.Errorf("could not migrate datastore: %w", err) }