Merge pull request #68520 from verult/top-auth-integration

Auth integration tests for CSINodeInfo changes

test/integration/auth/BUILD

@@ -19,6 +19,7 @@ go_test(
     ],
     tags = ["integration"],
     deps = [
+        "//cmd/kube-apiserver/app/testing:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/api/testapi:go_default_library",
         "//pkg/apis/authorization:go_default_library",
@@ -29,11 +30,9 @@ go_test(
         "//pkg/apis/policy:go_default_library",
         "//pkg/apis/rbac:go_default_library",
         "//pkg/auth/authorizer/abac:go_default_library",
-        "//pkg/auth/nodeidentifier:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/controller/serviceaccount:go_default_library",
         "//pkg/features:go_default_library",
-        "//pkg/kubeapiserver/authorizer:go_default_library",
         "//pkg/master:go_default_library",
         "//pkg/registry/rbac/clusterrole:go_default_library",
         "//pkg/registry/rbac/clusterrole/storage:go_default_library",
@@ -44,13 +43,14 @@ go_test(
         "//pkg/registry/rbac/rolebinding:go_default_library",
         "//pkg/registry/rbac/rolebinding/storage:go_default_library",
         "//pkg/serviceaccount:go_default_library",
-        "//plugin/pkg/admission/noderestriction:go_default_library",
         "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
         "//plugin/pkg/auth/authorizer/rbac:go_default_library",
         "//staging/src/k8s.io/api/authentication/v1:go_default_library",
         "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -72,7 +72,6 @@ go_test(
         "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
         "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/tokentest:go_default_library",
         "//staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/webhook:go_default_library",
-        "//staging/src/k8s.io/client-go/informers:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/client-go/tools/clientcmd/api/v1:go_default_library",
@@ -80,6 +79,9 @@ go_test(
         "//staging/src/k8s.io/client-go/transport:go_default_library",
         "//staging/src/k8s.io/client-go/util/cert:go_default_library",
         "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
+        "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
+        "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
+        "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
         "//test/e2e/lifecycle/bootstrap:go_default_library",
         "//test/integration:go_default_library",
         "//test/integration/framework:go_default_library",

test/integration/auth/node_test.go

@@ -18,46 +18,38 @@ package auth
 
 import (
 	"fmt"
-	"net/http"
-	"net/http/httptest"
 	"testing"
 	"time"
 
 	storagev1beta1 "k8s.io/api/storage/v1beta1"
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
-	"k8s.io/apiserver/pkg/authentication/token/tokenfile"
-	"k8s.io/apiserver/pkg/authentication/user"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing"
-	versionedinformers "k8s.io/client-go/informers"
 	externalclientset "k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
+	csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/apis/coordination"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/policy"
-	"k8s.io/kubernetes/pkg/auth/nodeidentifier"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/features"
-	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer"
-	"k8s.io/kubernetes/plugin/pkg/admission/noderestriction"
 	"k8s.io/kubernetes/test/integration/framework"
 	"k8s.io/utils/pointer"
+
+	"io/ioutil"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
+	csicrd "k8s.io/csi-api/pkg/crd"
+	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
+	"k8s.io/kubernetes/pkg/apis/core"
+	"strings"
 )
 
 func TestNodeAuthorizer(t *testing.T) {
-	// Start the server so we know the address
-	h := &framework.MasterHolder{Initialized: make(chan struct{})}
-	apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		<-h.Initialized
-		h.M.GenericAPIServer.Handler.ServeHTTP(w, req)
-	}))
-
 	const (
 		// Define credentials
 		tokenMaster      = "master-token"
@@ -66,18 +58,6 @@ func TestNodeAuthorizer(t *testing.T) {
 		tokenNode2       = "node2-token"
 	)
 
-	authenticator := bearertoken.New(tokenfile.New(map[string]*user.DefaultInfo{
-		tokenMaster:      {Name: "admin", Groups: []string{"system:masters"}},
-		tokenNodeUnknown: {Name: "unknown", Groups: []string{"system:nodes"}},
-		tokenNode1:       {Name: "system:node:node1", Groups: []string{"system:nodes"}},
-		tokenNode2:       {Name: "system:node:node2", Groups: []string{"system:nodes"}},
-	}))
-
-	// Build client config, clientset, and informers
-	clientConfig := &restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{NegotiatedSerializer: legacyscheme.Codecs}}
-	superuserClient, superuserClientExternal := clientsetForToken(tokenMaster, clientConfig)
-	versionedInformerFactory := versionedinformers.NewSharedInformerFactory(superuserClientExternal, time.Minute)
-
 	// Enabled CSIPersistentVolume feature at startup so volumeattachments get watched
 	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIPersistentVolume, true)()
 
@@ -87,36 +67,35 @@ func TestNodeAuthorizer(t *testing.T) {
 	// Enable NodeLease feature so that nodes can create leases
 	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeLease, true)()
 
-	// Set up Node+RBAC authorizer
+	// Enable CSINodeInfo feature so that nodes can create CSINodeInfo objects.
-	authorizerConfig := &authorizer.AuthorizationConfig{
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSINodeInfo, true)()
-		AuthorizationModes:       []string{"Node", "RBAC"},
+
-		VersionedInformerFactory: versionedInformerFactory,
+	tokenFile, err := ioutil.TempFile("", "kubeconfig")
-	}
-	nodeRBACAuthorizer, _, err := authorizerConfig.New()
 	if err != nil {
 		t.Fatal(err)
 	}
+	tokenFile.WriteString(strings.Join([]string{
+		fmt.Sprintf(`%s,admin,uid1,"system:masters"`, tokenMaster),
+		fmt.Sprintf(`%s,unknown,uid2,"system:nodes"`, tokenNodeUnknown),
+		fmt.Sprintf(`%s,system:node:node1,uid3,"system:nodes"`, tokenNode1),
+		fmt.Sprintf(`%s,system:node:node2,uid4,"system:nodes"`, tokenNode2),
+	}, "\n"))
+	tokenFile.Close()
 
-	// Set up NodeRestriction admission
+	server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{
-	nodeRestrictionAdmission := noderestriction.NewPlugin(nodeidentifier.NewDefaultNodeIdentifier())
+		"--authorization-mode", "Node,RBAC",
-	nodeRestrictionAdmission.SetExternalKubeInformerFactory(versionedInformerFactory)
+		"--token-auth-file", tokenFile.Name(),
-	if err := nodeRestrictionAdmission.ValidateInitialization(); err != nil {
+		"--enable-admission-plugins", "NodeRestriction",
-		t.Fatal(err)
+		// The "default" SA is not installed, causing the ServiceAccount plugin to retry for ~1s per
-	}
+		// API request.
+		"--disable-admission-plugins", "ServiceAccount",
+	}, framework.SharedEtcd())
+	defer server.TearDownFn()
 
-	// Start the server
+	// Build client config and superuser clientset
-	masterConfig := framework.NewIntegrationTestMasterConfig()
+	clientConfig := server.ClientConfig
-	masterConfig.GenericConfig.Authentication.Authenticator = authenticator
+	superuserClient, superuserClientExternal := clientsetForToken(tokenMaster, clientConfig)
-	masterConfig.GenericConfig.Authorization.Authorizer = nodeRBACAuthorizer
+	superuserCRDClient := crdClientsetForToken(tokenMaster, clientConfig)
-	masterConfig.GenericConfig.AdmissionControl = nodeRestrictionAdmission
-
-	_, _, closeFn := framework.RunAMasterUsingServer(masterConfig, apiServer, h)
-	defer closeFn()
-
-	// Start the informers
-	stopCh := make(chan struct{})
-	defer close(stopCh)
-	versionedInformerFactory.Start(stopCh)
 
 	// Wait for a healthy server
 	for {
@@ -130,6 +109,10 @@ func TestNodeAuthorizer(t *testing.T) {
 	}
 
 	// Create objects
+	if _, err := superuserClient.Core().Namespaces().Create(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns"}}); err != nil {
+		t.Fatal(err)
+	}
+
 	if _, err := superuserClient.Core().Secrets("ns").Create(&api.Secret{ObjectMeta: metav1.ObjectMeta{Name: "mysecret"}}); err != nil {
 		t.Fatal(err)
 	}
@@ -175,6 +158,14 @@ func TestNodeAuthorizer(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	crd, err := superuserCRDClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(csicrd.CSINodeInfoCRD())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForEstablishedCRD(superuserCRDClient, crd.Name); err != nil {
+		t.Fatalf("Failed to establish CSINodeInfo CRD: %v", err)
+	}
+
 	getSecret := func(client clientset.Interface) func() error {
 		return func() error {
 			_, err := client.Core().Secrets("ns").Get("mysecret", metav1.GetOptions{})
@@ -418,9 +409,66 @@ func TestNodeAuthorizer(t *testing.T) {
 		}
 	}
 
+	getNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
+		return func() error {
+			_, err := client.CsiV1alpha1().CSINodeInfos().Get("node1", metav1.GetOptions{})
+			return err
+		}
+	}
+	createNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
+		return func() error {
+			nodeInfo := &csiv1alpha1.CSINodeInfo{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "node1",
+				},
+				CSIDrivers: []csiv1alpha1.CSIDriverInfo{
+					{
+						Driver:       "com.example.csi/driver1",
+						NodeID:       "com.example.csi/node1",
+						TopologyKeys: []string{"com.example.csi/zone"},
+					},
+				},
+			}
+			_, err := client.CsiV1alpha1().CSINodeInfos().Create(nodeInfo)
+			return err
+		}
+	}
+	updateNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
+		return func() error {
+			nodeInfo, err := client.CsiV1alpha1().CSINodeInfos().Get("node1", metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			nodeInfo.CSIDrivers = []csiv1alpha1.CSIDriverInfo{
+				{
+					Driver:       "com.example.csi/driver1",
+					NodeID:       "com.example.csi/node1",
+					TopologyKeys: []string{"com.example.csi/rack"},
+				},
+			}
+			_, err = client.CsiV1alpha1().CSINodeInfos().Update(nodeInfo)
+			return err
+		}
+	}
+	patchNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
+		return func() error {
+			bs := []byte(fmt.Sprintf(`{"csiDrivers": [ { "driver": "net.example.storage/driver2", "nodeID": "net.example.storage/node1", "topologyKeys": [ "net.example.storage/region" ] } ] }`))
+			// StrategicMergePatch is unsupported by CRs. Falling back to MergePatch
+			_, err := client.CsiV1alpha1().CSINodeInfos().Patch("node1", types.MergePatchType, bs)
+			return err
+		}
+	}
+	deleteNode1CSINodeInfo := func(client csiclientset.Interface) func() error {
+		return func() error {
+			return client.CsiV1alpha1().CSINodeInfos().Delete("node1", &metav1.DeleteOptions{})
+		}
+	}
+
 	nodeanonClient, _ := clientsetForToken(tokenNodeUnknown, clientConfig)
 	node1Client, node1ClientExternal := clientsetForToken(tokenNode1, clientConfig)
 	node2Client, node2ClientExternal := clientsetForToken(tokenNode2, clientConfig)
+	csiNode1Client := csiClientsetForToken(tokenNode1, clientConfig)
+	csiNode2Client := csiClientsetForToken(tokenNode2, clientConfig)
 
 	// all node requests from node1 and unknown node fail
 	expectForbidden(t, getSecret(nodeanonClient))
@@ -573,7 +621,18 @@ func TestNodeAuthorizer(t *testing.T) {
 	expectForbidden(t, patchNode1Lease(node2Client))
 	expectForbidden(t, deleteNode1Lease(node2Client))
 
-	// TODO (verult) CSINodeInfo tests (issue #68254)
+	// node1 allowed to operate on its own CSINodeInfo
+	expectAllowed(t, createNode1CSINodeInfo(csiNode1Client))
+	expectAllowed(t, getNode1CSINodeInfo(csiNode1Client))
+	expectAllowed(t, updateNode1CSINodeInfo(csiNode1Client))
+	expectAllowed(t, patchNode1CSINodeInfo(csiNode1Client))
+	expectAllowed(t, deleteNode1CSINodeInfo(csiNode1Client))
+	// node2 not allowed to operate on another node's CSINodeInfo
+	expectForbidden(t, createNode1CSINodeInfo(csiNode2Client))
+	expectForbidden(t, getNode1CSINodeInfo(csiNode2Client))
+	expectForbidden(t, updateNode1CSINodeInfo(csiNode2Client))
+	expectForbidden(t, patchNode1CSINodeInfo(csiNode2Client))
+	expectForbidden(t, deleteNode1CSINodeInfo(csiNode2Client))
 }
 
 // expect executes a function a set number of times until it either returns the
@@ -613,3 +672,25 @@ func expectAllowed(t *testing.T, f func() error) {
 		t.Errorf("Expected no error, got %v", err)
 	}
 }
+
+func waitForEstablishedCRD(client apiextensionsclientset.Interface, name string) error {
+	return wait.PollImmediate(500*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
+		crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, cond := range crd.Status.Conditions {
+			switch cond.Type {
+			case apiextensionsv1beta1.Established:
+				if cond.Status == apiextensionsv1beta1.ConditionTrue {
+					return true, err
+				}
+			case apiextensionsv1beta1.NamesAccepted:
+				if cond.Status == apiextensionsv1beta1.ConditionFalse {
+					fmt.Printf("Name conflict: %v\n", cond.Reason)
+				}
+			}
+		}
+		return false, nil
+	})
+}

test/integration/auth/rbac_test.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/golang/glog"
 
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/watch"
@@ -41,6 +42,7 @@ import (
 	restclient "k8s.io/client-go/rest"
 	watchtools "k8s.io/client-go/tools/watch"
 	"k8s.io/client-go/transport"
+	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	api "k8s.io/kubernetes/pkg/apis/core"
@@ -74,6 +76,19 @@ func clientsetForToken(user string, config *restclient.Config) (clientset.Interf
 	return clientset.NewForConfigOrDie(&configCopy), externalclientset.NewForConfigOrDie(&configCopy)
 }
 
+func crdClientsetForToken(user string, config *restclient.Config) apiextensionsclient.Interface {
+	configCopy := *config
+	configCopy.BearerToken = user
+	return apiextensionsclient.NewForConfigOrDie(&configCopy)
+}
+
+func csiClientsetForToken(user string, config *restclient.Config) csiclientset.Interface {
+	configCopy := *config
+	configCopy.BearerToken = user
+	configCopy.ContentType = "application/json" // // csi client works with CRDs that support json only
+	return csiclientset.NewForConfigOrDie(&configCopy)
+}
+
 type testRESTOptionsGetter struct {
 	config *master.Config
 }