diff --git a/main.go b/main.go
index cae48b9..1ae59f4 100644
--- a/main.go
+++ b/main.go
@@ -35,6 +35,6 @@ func main() {
 func run(_ *cli.Context) error {
 	ctx := signals.SetupSignalHandler(context.Background())
 	debugconfig.MustSetupDebug()
-	s := config.MustServerConfig().MustServer()
-	return s.ListenAndServe(ctx, nil)
+	s := config.MustServer()
+	return s.ListenAndServe(ctx, config.HTTPSListenPort, config.HTTPListenPort, nil)
 }
diff --git a/pkg/server/cli/clicontext.go b/pkg/server/cli/clicontext.go
index bc68ca9..170e870 100644
--- a/pkg/server/cli/clicontext.go
+++ b/pkg/server/cli/clicontext.go
@@ -16,15 +16,15 @@ type Config struct {
 	WebhookConfig authcli.WebhookConfig
 }
 
-func (c *Config) MustServerConfig() *server.Server {
-	cc, err := c.ToServerConfig()
+func (c *Config) MustServer() *server.Server {
+	cc, err := c.ToServer()
 	if err != nil {
 		panic(err)
 	}
 	return cc
 }
 
-func (c *Config) ToServerConfig() (*server.Server, error) {
+func (c *Config) ToServer() (*server.Server, error) {
 	restConfig, err := kubeconfig.GetNonInteractiveClientConfig(c.KubeConfig).ClientConfig()
 	if err != nil {
 		return nil, err
@@ -39,8 +39,6 @@ func (c *Config) ToServerConfig() (*server.Server, error) {
 		Namespace:      c.Namespace,
 		RestConfig:     restConfig,
 		AuthMiddleware: auth,
-		HTTPPort:       c.HTTPListenPort,
-		HTTPSPort:      c.HTTPSListenPort,
 	}, nil
 }
 
diff --git a/pkg/server/config.go b/pkg/server/config.go
index 3fb7815..732b08e 100644
--- a/pkg/server/config.go
+++ b/pkg/server/config.go
@@ -27,8 +27,6 @@ type Server struct {
 	RestConfig *rest.Config
 
 	Namespace       string
-	HTTPSPort       int
-	HTTPPort        int
 	BaseSchemas     *types.APISchemas
 	SchemaTemplates []schema.Template
 	AuthMiddleware  auth.Middleware
@@ -47,6 +45,10 @@ type Controllers struct {
 	starters []start.Starter
 }
 
+func (c *Controllers) Start(ctx context.Context) error {
+	return start.All(ctx, 5, c.starters...)
+}
+
 func NewController(cfg *rest.Config) (*Controllers, error) {
 	c := &Controllers{}
 
diff --git a/pkg/server/server.go b/pkg/server/server.go
index d2ea3dd..95c90d9 100644
--- a/pkg/server/server.go
+++ b/pkg/server/server.go
@@ -6,8 +6,6 @@ import (
 	"net/http"
 
 	"github.com/rancher/dynamiclistener/server"
-	"github.com/rancher/dynamiclistener/storage/kubernetes"
-	"github.com/rancher/dynamiclistener/storage/memory"
 	"github.com/rancher/steve/pkg/accesscontrol"
 	"github.com/rancher/steve/pkg/client"
 	"github.com/rancher/steve/pkg/clustercache"
@@ -16,8 +14,6 @@ import (
 	"github.com/rancher/steve/pkg/schemaserver/types"
 	"github.com/rancher/steve/pkg/server/handler"
 	"github.com/rancher/steve/pkg/server/resources"
-	v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
-	"github.com/rancher/wrangler/pkg/start"
 )
 
 var ErrConfigRequired = errors.New("rest config is required")
@@ -102,7 +98,7 @@ func (c *Server) Handler(ctx context.Context) (http.Handler, error) {
 		sf.AddTemplate(&c.SchemaTemplates[i])
 	}
 
-	if err := start.All(ctx, 5, c.starters...); err != nil {
+	if err := c.Controllers.Start(ctx); err != nil {
 		return nil, err
 	}
 
@@ -115,39 +111,26 @@ func (c *Server) Handler(ctx context.Context) (http.Handler, error) {
 	return handler, nil
 }
 
-func ListenAndServe(ctx context.Context, secrets v1.SecretController, namespace string, handler http.Handler, httpsPort, httpPort int, opts *server.ListenOpts) error {
-	var (
-		err error
-	)
-
-	if opts == nil {
-		opts = &server.ListenOpts{}
-	}
-
-	if opts.CA == nil || opts.CAKey == nil {
-		opts.CA, opts.CAKey, err = kubernetes.LoadOrGenCA(secrets, namespace, "serving-ca")
-		if err != nil {
-			return err
-		}
-	}
-
-	if opts.Storage == nil {
-		storage := kubernetes.Load(ctx, secrets, namespace, "service-cert", memory.New())
-		opts.Storage = storage
-	}
-
-	if err := server.ListenAndServe(ctx, httpsPort, httpPort, handler, opts); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (c *Server) ListenAndServe(ctx context.Context, httpsPort, httpPort int, opts *server.ListenOpts) error {
 	handler, err := c.Handler(ctx)
 	if err != nil {
 		return err
 	}
 
-	return ListenAndServe(ctx, c.Core.Secret(), c.Namespace, handler, httpsPort, httpPort, opts)
+	if opts == nil {
+		opts = &server.ListenOpts{}
+	}
+	if opts.Storage == nil && opts.Secrets == nil {
+		opts.Secrets = c.Core.Secret()
+	}
+	if err := server.ListenAndServe(ctx, httpsPort, httpPort, handler, opts); err != nil {
+		return err
+	}
+
+	if err := c.Controllers.Start(ctx); err != nil {
+		return err
+	}
+
+	<-ctx.Done()
+	return ctx.Err()
 }