diff --git a/test/integration/auth/BUILD b/test/integration/auth/BUILD index a5d43877727..6d4d735bdab 100644 --- a/test/integration/auth/BUILD +++ b/test/integration/auth/BUILD @@ -19,6 +19,7 @@ go_test( ], tags = ["integration"], deps = [ + "//cmd/kube-apiserver/app/testing:go_default_library", "//pkg/api/legacyscheme:go_default_library", "//pkg/api/testapi:go_default_library", "//pkg/apis/authorization:go_default_library", @@ -29,11 +30,9 @@ go_test( "//pkg/apis/policy:go_default_library", "//pkg/apis/rbac:go_default_library", "//pkg/auth/authorizer/abac:go_default_library", - "//pkg/auth/nodeidentifier:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/controller/serviceaccount:go_default_library", "//pkg/features:go_default_library", - "//pkg/kubeapiserver/authorizer:go_default_library", "//pkg/master:go_default_library", "//pkg/registry/rbac/clusterrole:go_default_library", "//pkg/registry/rbac/clusterrole/storage:go_default_library", @@ -44,13 +43,14 @@ go_test( "//pkg/registry/rbac/rolebinding:go_default_library", "//pkg/registry/rbac/rolebinding/storage:go_default_library", "//pkg/serviceaccount:go_default_library", - "//plugin/pkg/admission/noderestriction:go_default_library", "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library", "//plugin/pkg/auth/authorizer/rbac:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -72,7 +72,6 @@ go_test( "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest:go_default_library", "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook:go_default_library", - "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/rest:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api/v1:go_default_library", @@ -80,6 +79,9 @@ go_test( "//staging/src/k8s.io/client-go/transport:go_default_library", "//staging/src/k8s.io/client-go/util/cert:go_default_library", "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library", "//test/e2e/lifecycle/bootstrap:go_default_library", "//test/integration:go_default_library", "//test/integration/framework:go_default_library", diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index 6f935824470..db871d6d37b 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -18,46 +18,38 @@ package auth import ( "fmt" - "net/http" - "net/http/httptest" "testing" "time" storagev1beta1 "k8s.io/api/storage/v1beta1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/authentication/request/bearertoken" - "k8s.io/apiserver/pkg/authentication/token/tokenfile" - "k8s.io/apiserver/pkg/authentication/user" utilfeature "k8s.io/apiserver/pkg/util/feature" utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" - versionedinformers "k8s.io/client-go/informers" externalclientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/kubernetes/pkg/api/legacyscheme" + csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" "k8s.io/kubernetes/pkg/apis/coordination" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" - "k8s.io/kubernetes/pkg/auth/nodeidentifier" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/kubeapiserver/authorizer" - "k8s.io/kubernetes/plugin/pkg/admission/noderestriction" "k8s.io/kubernetes/test/integration/framework" "k8s.io/utils/pointer" + + "io/ioutil" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + csicrd "k8s.io/csi-api/pkg/crd" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/pkg/apis/core" + "strings" ) func TestNodeAuthorizer(t *testing.T) { - // Start the server so we know the address - h := &framework.MasterHolder{Initialized: make(chan struct{})} - apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - <-h.Initialized - h.M.GenericAPIServer.Handler.ServeHTTP(w, req) - })) - const ( // Define credentials tokenMaster = "master-token" @@ -66,18 +58,6 @@ func TestNodeAuthorizer(t *testing.T) { tokenNode2 = "node2-token" ) - authenticator := bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{ - tokenMaster: {Name: "admin", Groups: []string{"system:masters"}}, - tokenNodeUnknown: {Name: "unknown", Groups: []string{"system:nodes"}}, - tokenNode1: {Name: "system:node:node1", Groups: []string{"system:nodes"}}, - tokenNode2: {Name: "system:node:node2", Groups: []string{"system:nodes"}}, - })) - - // Build client config, clientset, and informers - clientConfig := &restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}} - superuserClient, superuserClientExternal := clientsetForToken(tokenMaster, clientConfig) - versionedInformerFactory := versionedinformers.NewSharedInformerFactory(superuserClientExternal, time.Minute) - // Enabled CSIPersistentVolume feature at startup so volumeattachments get watched defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIPersistentVolume, true)() @@ -87,36 +67,35 @@ func TestNodeAuthorizer(t *testing.T) { // Enable NodeLease feature so that nodes can create leases defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeLease, true)() - // Set up Node+RBAC authorizer - authorizerConfig := &authorizer.AuthorizationConfig{ - AuthorizationModes: []string{"Node", "RBAC"}, - VersionedInformerFactory: versionedInformerFactory, - } - nodeRBACAuthorizer, _, err := authorizerConfig.New() + // Enable CSINodeInfo feature so that nodes can create CSINodeInfo objects. + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSINodeInfo, true)() + + tokenFile, err := ioutil.TempFile("", "kubeconfig") if err != nil { t.Fatal(err) } + tokenFile.WriteString(strings.Join([]string{ + fmt.Sprintf(`%s,admin,uid1,"system:masters"`, tokenMaster), + fmt.Sprintf(`%s,unknown,uid2,"system:nodes"`, tokenNodeUnknown), + fmt.Sprintf(`%s,system:node:node1,uid3,"system:nodes"`, tokenNode1), + fmt.Sprintf(`%s,system:node:node2,uid4,"system:nodes"`, tokenNode2), + }, "\n")) + tokenFile.Close() - // Set up NodeRestriction admission - nodeRestrictionAdmission := noderestriction.NewPlugin(nodeidentifier.NewDefaultNodeIdentifier()) - nodeRestrictionAdmission.SetExternalKubeInformerFactory(versionedInformerFactory) - if err := nodeRestrictionAdmission.ValidateInitialization(); err != nil { - t.Fatal(err) - } + server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{ + "--authorization-mode", "Node,RBAC", + "--token-auth-file", tokenFile.Name(), + "--enable-admission-plugins", "NodeRestriction", + // The "default" SA is not installed, causing the ServiceAccount plugin to retry for ~1s per + // API request. + "--disable-admission-plugins", "ServiceAccount", + }, framework.SharedEtcd()) + defer server.TearDownFn() - // Start the server - masterConfig := framework.NewIntegrationTestMasterConfig() - masterConfig.GenericConfig.Authentication.Authenticator = authenticator - masterConfig.GenericConfig.Authorization.Authorizer = nodeRBACAuthorizer - masterConfig.GenericConfig.AdmissionControl = nodeRestrictionAdmission - - _, _, closeFn := framework.RunAMasterUsingServer(masterConfig, apiServer, h) - defer closeFn() - - // Start the informers - stopCh := make(chan struct{}) - defer close(stopCh) - versionedInformerFactory.Start(stopCh) + // Build client config and superuser clientset + clientConfig := server.ClientConfig + superuserClient, superuserClientExternal := clientsetForToken(tokenMaster, clientConfig) + superuserCRDClient := crdClientsetForToken(tokenMaster, clientConfig) // Wait for a healthy server for { @@ -130,6 +109,10 @@ func TestNodeAuthorizer(t *testing.T) { } // Create objects + if _, err := superuserClient.Core().Namespaces().Create(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns"}}); err != nil { + t.Fatal(err) + } + if _, err := superuserClient.Core().Secrets("ns").Create(&api.Secret{ObjectMeta: metav1.ObjectMeta{Name: "mysecret"}}); err != nil { t.Fatal(err) } @@ -175,6 +158,14 @@ func TestNodeAuthorizer(t *testing.T) { t.Fatal(err) } + crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD()) + if err != nil { + t.Fatal(err) + } + if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil { + t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err) + } + getSecret := func(client clientset.Interface) func() error { return func() error { _, err := client.Core().Secrets("ns").Get("mysecret", metav1.GetOptions{}) @@ -418,9 +409,66 @@ func TestNodeAuthorizer(t *testing.T) { } } + getNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + return func() error { + _, err := client.CsiV1alpha1().CSINodeInfos().Get("node1", metav1.GetOptions{}) + return err + } + } + createNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + return func() error { + nodeInfo := &csiv1alpha1.CSINodeInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + CSIDrivers: []csiv1alpha1.CSIDriverInfo{ + { + Driver: "com.example.csi/driver1", + NodeID: "com.example.csi/node1", + TopologyKeys: []string{"com.example.csi/zone"}, + }, + }, + } + _, err := client.CsiV1alpha1().CSINodeInfos().Create(nodeInfo) + return err + } + } + updateNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + return func() error { + nodeInfo, err := client.CsiV1alpha1().CSINodeInfos().Get("node1", metav1.GetOptions{}) + if err != nil { + return err + } + nodeInfo.CSIDrivers = []csiv1alpha1.CSIDriverInfo{ + { + Driver: "com.example.csi/driver1", + NodeID: "com.example.csi/node1", + TopologyKeys: []string{"com.example.csi/rack"}, + }, + } + _, err = client.CsiV1alpha1().CSINodeInfos().Update(nodeInfo) + return err + } + } + patchNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + return func() error { + bs := []byte(fmt.Sprintf(`{"csiDrivers": [ { "driver": "net.example.storage/driver2", "nodeID": "net.example.storage/node1", "topologyKeys": [ "net.example.storage/region" ] } ] }`)) + // StrategicMergePatch is unsupported by CRs. Falling back to MergePatch + _, err := client.CsiV1alpha1().CSINodeInfos().Patch("node1", types.MergePatchType, bs) + return err + } + } + deleteNode1CSINodeInfo := func(client csiclientset.Interface) func() error { + return func() error { + return client.CsiV1alpha1().CSINodeInfos().Delete("node1", &metav1.DeleteOptions{}) + } + } + nodeanonClient, _ := clientsetForToken(tokenNodeUnknown, clientConfig) node1Client, node1ClientExternal := clientsetForToken(tokenNode1, clientConfig) node2Client, node2ClientExternal := clientsetForToken(tokenNode2, clientConfig) + csiNode1Client := csiClientsetForToken(tokenNode1, clientConfig) + csiNode2Client := csiClientsetForToken(tokenNode2, clientConfig) // all node requests from node1 and unknown node fail expectForbidden(t, getSecret(nodeanonClient)) @@ -573,7 +621,18 @@ func TestNodeAuthorizer(t *testing.T) { expectForbidden(t, patchNode1Lease(node2Client)) expectForbidden(t, deleteNode1Lease(node2Client)) - // TODO (verult) CSINodeInfo tests (issue #68254) + // node1 allowed to operate on its own CSINodeInfo + expectAllowed(t, createNode1CSINodeInfo(csiNode1Client)) + expectAllowed(t, getNode1CSINodeInfo(csiNode1Client)) + expectAllowed(t, updateNode1CSINodeInfo(csiNode1Client)) + expectAllowed(t, patchNode1CSINodeInfo(csiNode1Client)) + expectAllowed(t, deleteNode1CSINodeInfo(csiNode1Client)) + // node2 not allowed to operate on another node's CSINodeInfo + expectForbidden(t, createNode1CSINodeInfo(csiNode2Client)) + expectForbidden(t, getNode1CSINodeInfo(csiNode2Client)) + expectForbidden(t, updateNode1CSINodeInfo(csiNode2Client)) + expectForbidden(t, patchNode1CSINodeInfo(csiNode2Client)) + expectForbidden(t, deleteNode1CSINodeInfo(csiNode2Client)) } // expect executes a function a set number of times until it either returns the @@ -613,3 +672,25 @@ func expectAllowed(t *testing.T, f func() error) { t.Errorf("Expected no error, got %v", err) } } + +func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error { + return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) + if err != nil { + return false, err + } + for _, cond := range crd.Status.Conditions { + switch cond.Type { + case apiextensionsv1beta1.Established: + if cond.Status == apiextensionsv1beta1.ConditionTrue { + return true, err + } + case apiextensionsv1beta1.NamesAccepted: + if cond.Status == apiextensionsv1beta1.ConditionFalse { + fmt.Printf("Name conflict: %v\n", cond.Reason) + } + } + } + return false, nil + }) +} diff --git a/test/integration/auth/rbac_test.go b/test/integration/auth/rbac_test.go index 0c386e4d975..9c6865c9e9b 100644 --- a/test/integration/auth/rbac_test.go +++ b/test/integration/auth/rbac_test.go @@ -29,6 +29,7 @@ import ( "github.com/golang/glog" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/watch" @@ -41,6 +42,7 @@ import ( restclient "k8s.io/client-go/rest" watchtools "k8s.io/client-go/tools/watch" "k8s.io/client-go/transport" + csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/api/testapi" api "k8s.io/kubernetes/pkg/apis/core" @@ -74,6 +76,19 @@ func clientsetForToken(user string, config *restclient.Config) (clientset.Interf return clientset.NewForConfigOrDie(&configCopy), externalclientset.NewForConfigOrDie(&configCopy) } +func crdClientsetForToken(user string, config *restclient.Config) apiextensionsclient.Interface { + configCopy := *config + configCopy.BearerToken = user + return apiextensionsclient.NewForConfigOrDie(&configCopy) +} + +func csiClientsetForToken(user string, config *restclient.Config) csiclientset.Interface { + configCopy := *config + configCopy.BearerToken = user + configCopy.ContentType = "application/json" // // csi client works with CRDs that support json only + return csiclientset.NewForConfigOrDie(&configCopy) +} + type testRESTOptionsGetter struct { config *master.Config }