Merge pull request #39094 from deads2k/rbac-32-finish-e2es

Automatic merge from submit-queue (batch tested with PRs 39146, 39094)

cleanup last e2e authorization failures

Builds on https://github.com/kubernetes/kubernetes/pull/39080.  This adds rbac role bindings during e2e tests for test that use SA permissions to loopback to the API server.

Assigned to me until its ready.

cluster/addons/e2e-rbac-bindings/README.md

@@ -0,0 +1,3 @@
+These resources are used to add extra (non-default) bindings to e2e to match users and groups
+that are particular to the e2e environment.  These are not standard bootstrap bindings and 
+not standard users they are bound to.  This is not a recipe for adding bootstrap bindings.

cluster/addons/e2e-rbac-bindings/admin-binding.yaml

@@ -0,0 +1,16 @@
+# something in the kube e2e uses an admin identity to list pods
+# TODO figure out what is doing this and ultimately remove this binding
+apiVersion: rbac.authorization.k8s.io/v1alpha1
+kind: ClusterRoleBinding
+metadata:
+  name: admin-cluster-admin
+  labels:
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: view
+subjects:
+- apiVersion: rbac/v1alpha1
+  kind: User
+  name: admin

cluster/addons/e2e-rbac-bindings/kubecfg-binding.yaml

@@ -0,0 +1,18 @@
+# This is the main user for the e2e tests.  This is ok to leave long term
+# since the first user in the test can reasonably be high power
+# TODO consider provisioning each test its namespace and giving it an
+# admin user.  This still has to exist, but e2e wouldn't normally use it
+apiVersion: rbac.authorization.k8s.io/v1alpha1
+kind: ClusterRoleBinding
+metadata:
+  name: kubecfg-cluster-admin
+  labels:
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- apiVersion: rbac/v1alpha1
+  kind: User
+  name: kubecfg

cluster/addons/e2e-rbac-bindings/kubelet-binding.yaml

@@ -0,0 +1,19 @@
+# The GKE environments don't have kubelets with certificates that
+# identify the system:nodes group.  They use the kubelet identity
+# TODO cjcullen should figure out how wants to manage his upgrade
+# this will only hold the e2e tests until we get an authorizer
+# which authorizes particular nodes
+apiVersion: rbac.authorization.k8s.io/v1alpha1
+kind: ClusterRoleBinding
+metadata:
+  name: kubelet-cluster-admin
+  labels:
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- apiVersion: rbac/v1alpha1
+  kind: User
+  name: kubelet

cluster/addons/e2e-rbac-bindings/random-addon-grabbag.yaml

@@ -0,0 +1,19 @@
+# TODO remove this
+# currently, the kube-addon-manager is adding lots of pods which all share
+# the system:serviceaccount:kube-system:default identity.  We need to subdivide
+# those service accounts, figure out which ones we're going to make bootstrap roles for
+# and bind those particular roles in the addon yaml itself.  This just gets us started
+apiVersion: rbac.authorization.k8s.io/v1alpha1
+kind: ClusterRoleBinding
+metadata:
+  name: todo-remove-grabbag-cluster-admin
+  labels:
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: kube-system

cluster/gce/gci/configure-helper.sh

@@ -1058,6 +1058,10 @@ function start-kube-addons {
   echo "Prepare kube-addons manifests and start kube addon manager"
   local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty"
   local -r dst_dir="/etc/kubernetes/addons"
+
+  # prep the additional bindings that are particular to e2e users and groups
+  setup-addon-manifests "addons" "e2e-rbac-bindings"
+
   # Set up manifests of other addons.
   if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \
      [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \

test/e2e/BUILD

@@ -128,6 +128,7 @@ go_library(
         "//pkg/apis/extensions:go_default_library",
         "//pkg/apis/extensions/v1beta1:go_default_library",
         "//pkg/apis/meta/v1:go_default_library",
+        "//pkg/apis/rbac/v1alpha1:go_default_library",
         "//pkg/apis/storage/util:go_default_library",
         "//pkg/apis/storage/v1beta1:go_default_library",
         "//pkg/apis/storage/v1beta1/util:go_default_library",
@@ -163,6 +164,7 @@ go_library(
         "//pkg/runtime:go_default_library",
         "//pkg/runtime/schema:go_default_library",
         "//pkg/security/apparmor:go_default_library",
+        "//pkg/serviceaccount:go_default_library",
         "//pkg/types:go_default_library",
         "//pkg/util:go_default_library",
         "//pkg/util/exec:go_default_library",

test/e2e/framework/BUILD

@@ -10,6 +10,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "authorizer_util.go",
         "cleanup.go",
         "exec_util.go",
         "federation_util.go",
@@ -38,6 +39,7 @@ go_library(
         "//pkg/api/validation:go_default_library",
         "//pkg/apimachinery/registered:go_default_library",
         "//pkg/apis/apps/v1beta1:go_default_library",
+        "//pkg/apis/authorization/v1beta1:go_default_library",
         "//pkg/apis/batch:go_default_library",
         "//pkg/apis/batch/v1:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
@@ -46,6 +48,7 @@ go_library(
         "//pkg/apis/meta/v1:go_default_library",
         "//pkg/apis/meta/v1/unstructured:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//pkg/client/clientset_generated/clientset/typed/authorization/v1beta1:go_default_library",
         "//pkg/client/clientset_generated/clientset/typed/core/v1:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/conditions:go_default_library",

test/e2e/framework/authorizer_util.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"time"
+
+	authorizationv1beta1 "k8s.io/kubernetes/pkg/apis/authorization/v1beta1"
+	v1beta1authorization "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/authorization/v1beta1"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/util/wait"
+)
+
+const (
+	policyCachePollInterval = 100 * time.Millisecond
+	policyCachePollTimeout  = 5 * time.Second
+)
+
+// WaitForAuthorizationUpdate checks if the given user can perform the named verb and action.
+// If policyCachePollTimeout is reached without the expected condition matching, an error is returned
+func WaitForAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviewsGetter, user, namespace, verb string, resource schema.GroupResource, allowed bool) error {
+	review := &authorizationv1beta1.SubjectAccessReview{
+		Spec: authorizationv1beta1.SubjectAccessReviewSpec{
+			ResourceAttributes: &authorizationv1beta1.ResourceAttributes{
+				Group:     resource.Group,
+				Verb:      verb,
+				Resource:  resource.Resource,
+				Namespace: namespace,
+			},
+			User: user,
+		},
+	}
+	err := wait.Poll(policyCachePollInterval, policyCachePollTimeout, func() (bool, error) {
+		response, err := c.SubjectAccessReviews().Create(review)
+		if err != nil {
+			return false, err
+		}
+		if response.Status.Allowed != allowed {
+			return false, nil
+		}
+		return true, nil
+	})
+	return err
+}

test/e2e/ingress.go

@@ -21,6 +21,10 @@ import (
 	"path/filepath"
 	"time"
 
+	legacyv1 "k8s.io/kubernetes/pkg/api/v1"
+	rbacv1alpha1 "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
@@ -70,6 +74,32 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() {
 		f.BeforeEach()
 		jig = newTestJig(f.ClientSet)
 		ns = f.Namespace.Name
+
+		// this test wants powerful permissions.  Since the namespace names are unique, we can leave this
+		// lying around so we don't have to race any caches
+		_, err := jig.client.Rbac().ClusterRoleBindings().Create(&rbacv1alpha1.ClusterRoleBinding{
+			ObjectMeta: legacyv1.ObjectMeta{
+				Name: f.Namespace.Name + "--cluster-admin",
+			},
+			RoleRef: rbacv1alpha1.RoleRef{
+				APIGroup: "rbac.authorization.k8s.io",
+				Kind:     "ClusterRole",
+				Name:     "cluster-admin",
+			},
+			Subjects: []rbacv1alpha1.Subject{
+				{
+					Kind:      rbacv1alpha1.ServiceAccountKind,
+					Namespace: f.Namespace.Name,
+					Name:      "default",
+				},
+			},
+		})
+		framework.ExpectNoError(err)
+
+		err = framework.WaitForAuthorizationUpdate(jig.client.Authorization(),
+			serviceaccount.MakeUsername(f.Namespace.Name, "default"),
+			"", "create", schema.GroupResource{Resource: "pods"}, true)
+		framework.ExpectNoError(err)
 	})
 
 	// Before enabling this loadbalancer test in any other test list you must

test/e2e/node_problem_detector.go

@@ -24,11 +24,15 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
+	legacyv1 "k8s.io/kubernetes/pkg/api/v1"
 	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
+	rbacv1alpha1 "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	coreclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
 	"k8s.io/kubernetes/pkg/fields"
 	"k8s.io/kubernetes/pkg/labels"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/pkg/util/system"
 	"k8s.io/kubernetes/pkg/util/uuid"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -57,6 +61,32 @@ var _ = framework.KubeDescribe("NodeProblemDetector", func() {
 		configName = "node-problem-detector-config-" + uid
 		// There is no namespace for Node, event recorder will set default namespace for node events.
 		eventNamespace = v1.NamespaceDefault
+
+		// this test wants extra permissions.  Since the namespace names are unique, we can leave this
+		// lying around so we don't have to race any caches
+		_, err := f.ClientSet.Rbac().ClusterRoleBindings().Create(&rbacv1alpha1.ClusterRoleBinding{
+			ObjectMeta: legacyv1.ObjectMeta{
+				Name: f.Namespace.Name + "--cluster-admin",
+			},
+			RoleRef: rbacv1alpha1.RoleRef{
+				APIGroup: "rbac.authorization.k8s.io",
+				Kind:     "ClusterRole",
+				Name:     "cluster-admin",
+			},
+			Subjects: []rbacv1alpha1.Subject{
+				{
+					Kind:      rbacv1alpha1.ServiceAccountKind,
+					Namespace: f.Namespace.Name,
+					Name:      "default",
+				},
+			},
+		})
+		framework.ExpectNoError(err)
+
+		err = framework.WaitForAuthorizationUpdate(f.ClientSet.Authorization(),
+			serviceaccount.MakeUsername(f.Namespace.Name, "default"),
+			"", "create", schema.GroupResource{Resource: "pods"}, true)
+		framework.ExpectNoError(err)
 	})
 
 	// Test kernel monitor. We may add other tests if we have more problem daemons in the future.

test/e2e/pre_stop.go

@@ -22,8 +22,12 @@ import (
 	"time"
 
 	"k8s.io/kubernetes/pkg/api/v1"
+	legacyv1 "k8s.io/kubernetes/pkg/api/v1"
 	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
+	rbacv1alpha1 "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 
@@ -163,6 +167,34 @@ func testPreStop(c clientset.Interface, ns string) {
 var _ = framework.KubeDescribe("PreStop", func() {
 	f := framework.NewDefaultFramework("prestop")
 
+	BeforeEach(func() {
+		// this test wants extra permissions.  Since the namespace names are unique, we can leave this
+		// lying around so we don't have to race any caches
+		_, err := f.ClientSet.Rbac().ClusterRoleBindings().Create(&rbacv1alpha1.ClusterRoleBinding{
+			ObjectMeta: legacyv1.ObjectMeta{
+				Name: f.Namespace.Name + "--cluster-admin",
+			},
+			RoleRef: rbacv1alpha1.RoleRef{
+				APIGroup: "rbac.authorization.k8s.io",
+				Kind:     "ClusterRole",
+				Name:     "cluster-admin",
+			},
+			Subjects: []rbacv1alpha1.Subject{
+				{
+					Kind:      rbacv1alpha1.ServiceAccountKind,
+					Namespace: f.Namespace.Name,
+					Name:      "default",
+				},
+			},
+		})
+		framework.ExpectNoError(err)
+
+		err = framework.WaitForAuthorizationUpdate(f.ClientSet.Authorization(),
+			serviceaccount.MakeUsername(f.Namespace.Name, "default"),
+			"", "create", schema.GroupResource{Resource: "pods"}, true)
+		framework.ExpectNoError(err)
+	})
+
 	It("should call prestop when killing a pod [Conformance]", func() {
 		testPreStop(f.ClientSet, f.Namespace.Name)
 	})