unblock resources that the storage version manager depends on

to avoid deadlock itself

staging/src/k8s.io/apiserver/pkg/endpoints/filters/storageversion.go

@@ -22,8 +22,10 @@ import (
 	"net/http"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/storageversion"
@@ -36,7 +38,9 @@ import (
 // 1. non-resource requests,
 // 2. read requests,
 // 3. write requests to the storageversion API,
-// 4. resources whose StorageVersion is not pending update, including non-persisted resources.
+// 4. create requests to the namespace API sent by apiserver itself,
+// 5. write requests to the lease API in kube-system namespace,
+// 6. resources whose StorageVersion is not pending update, including non-persisted resources.
 func WithStorageVersionPrecondition(handler http.Handler, svm storageversion.Manager, s runtime.NegotiatedSerializer) http.Handler {
 	if svm == nil {
 		// TODO(roycaihw): switch to warning after the feature graduate to beta/GA
@@ -69,6 +73,31 @@ func WithStorageVersionPrecondition(handler http.Handler, svm storageversion.Man
 			handler.ServeHTTP(w, req)
 			return
 		}
+		// The system namespace is required for apiserver-identity lease to exist. Allow the apiserver
+		// itself to create namespaces.
+		// NOTE: with this exception, if the bootstrap client writes namespaces with a new version,
+		// and the upgraded apiserver dies before updating the StorageVersion for namespaces, the
+		// storage migrator won't be able to tell these namespaces are stored in a different version in etcd.
+		// Because the bootstrap client only creates system namespace and doesn't update them. This can
+		// only happen if the upgraded apiserver is the first apiserver that kicks off namespace creation,
+		// or if a upgraded server that joins an existing cluster have new system namespaces (other
+		// than kube-system, kube-public, kube-node-lease) that need to be created.
+		u, hasUser := request.UserFrom(ctx)
+		if requestInfo.APIGroup == "" && requestInfo.Resource == "namespaces" &&
+			requestInfo.Verb == "create" && hasUser &&
+			u.GetName() == user.APIServerUser && contains(u.GetGroups(), user.SystemPrivilegedGroup) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+		// Allow writes to the lease API in kube-system. The storage version API depends on the
+		// apiserver-identity leases to operate. Leases in kube-system are either apiserver-identity
+		// lease (which gets garbage collected when stale) or leader-election leases (which gets
+		// periodically updated by system components). Both types of leases won't be stale in etcd.
+		if requestInfo.APIGroup == "coordination.k8s.io" && requestInfo.Resource == "leases" &&
+			requestInfo.Namespace == metav1.NamespaceSystem {
+			handler.ServeHTTP(w, req)
+			return
+		}
 		// If the resource's StorageVersion is not in the to-be-updated list, let it pass.
 		// Non-persisted resources are not in the to-be-updated list, so they will pass.
 		gr := schema.GroupResource{requestInfo.APIGroup, requestInfo.Resource}
@@ -81,3 +110,12 @@ func WithStorageVersionPrecondition(handler http.Handler, svm storageversion.Man
 		responsewriters.ErrorNegotiated(apierrors.NewServiceUnavailable(fmt.Sprintf("wait for storage version registration to complete for resource: %v, last seen error: %v", gr, svm.LastUpdateError(gr))), s, gv, w, req)
 	})
 }
+
+func contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}

test/integration/storageversion/storage_version_filter_test.go

@@ -82,7 +82,7 @@ func assertBlocking(name string, t *testing.T, err error, shouldBlock bool) {
 
 func testBuiltinResourceWrite(t *testing.T, cfg *rest.Config, shouldBlock bool) {
 	client := clientset.NewForConfigOrDie(cfg)
-	_, err := client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, metav1.CreateOptions{})
+	_, err := client.CoreV1().ConfigMaps("default").Create(context.TODO(), &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, metav1.CreateOptions{})
 	assertBlocking("writes to built in resources", t, err, shouldBlock)
 }