diff --git a/cluster/addons/e2e-rbac-bindings/README.md b/cluster/addons/e2e-rbac-bindings/README.md new file mode 100644 index 00000000000..879bbb7dc80 --- /dev/null +++ b/cluster/addons/e2e-rbac-bindings/README.md @@ -0,0 +1,3 @@ +These resources are used to add extra (non-default) bindings to e2e to match users and groups +that are particular to the e2e environment. These are not standard bootstrap bindings and +not standard users they are bound to. This is not a recipe for adding bootstrap bindings. \ No newline at end of file diff --git a/cluster/addons/e2e-rbac-bindings/admin-binding.yaml b/cluster/addons/e2e-rbac-bindings/admin-binding.yaml new file mode 100644 index 00000000000..370635e7494 --- /dev/null +++ b/cluster/addons/e2e-rbac-bindings/admin-binding.yaml @@ -0,0 +1,16 @@ +# something in the kube e2e uses an admin identity to list pods +# TODO figure out what is doing this and ultimately remove this binding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRoleBinding +metadata: + name: admin-cluster-admin + labels: + kubernetes.io/cluster-service: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view +subjects: +- apiVersion: rbac/v1alpha1 + kind: User + name: admin diff --git a/cluster/addons/e2e-rbac-bindings/kubecfg-binding.yaml b/cluster/addons/e2e-rbac-bindings/kubecfg-binding.yaml new file mode 100644 index 00000000000..8bad467245a --- /dev/null +++ b/cluster/addons/e2e-rbac-bindings/kubecfg-binding.yaml @@ -0,0 +1,18 @@ +# This is the main user for the e2e tests. This is ok to leave long term +# since the first user in the test can reasonably be high power +# TODO consider provisioning each test its namespace and giving it an +# admin user. This still has to exist, but e2e wouldn't normally use it +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRoleBinding +metadata: + name: kubecfg-cluster-admin + labels: + kubernetes.io/cluster-service: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiVersion: rbac/v1alpha1 + kind: User + name: kubecfg diff --git a/cluster/addons/e2e-rbac-bindings/kubelet-binding.yaml b/cluster/addons/e2e-rbac-bindings/kubelet-binding.yaml new file mode 100644 index 00000000000..01e0b51f3cf --- /dev/null +++ b/cluster/addons/e2e-rbac-bindings/kubelet-binding.yaml @@ -0,0 +1,19 @@ +# The GKE environments don't have kubelets with certificates that +# identify the system:nodes group. They use the kubelet identity +# TODO cjcullen should figure out how wants to manage his upgrade +# this will only hold the e2e tests until we get an authorizer +# which authorizes particular nodes +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRoleBinding +metadata: + name: kubelet-cluster-admin + labels: + kubernetes.io/cluster-service: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiVersion: rbac/v1alpha1 + kind: User + name: kubelet diff --git a/cluster/addons/e2e-rbac-bindings/random-addon-grabbag.yaml b/cluster/addons/e2e-rbac-bindings/random-addon-grabbag.yaml new file mode 100644 index 00000000000..0e47f35d031 --- /dev/null +++ b/cluster/addons/e2e-rbac-bindings/random-addon-grabbag.yaml @@ -0,0 +1,19 @@ +# TODO remove this +# currently, the kube-addon-manager is adding lots of pods which all share +# the system:serviceaccount:kube-system:default identity. We need to subdivide +# those service accounts, figure out which ones we're going to make bootstrap roles for +# and bind those particular roles in the addon yaml itself. This just gets us started +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRoleBinding +metadata: + name: todo-remove-grabbag-cluster-admin + labels: + kubernetes.io/cluster-service: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: default + namespace: kube-system diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index b1536d83a25..543f53ac1de 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1058,6 +1058,10 @@ function start-kube-addons { echo "Prepare kube-addons manifests and start kube addon manager" local -r src_dir="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty" local -r dst_dir="/etc/kubernetes/addons" + + # prep the additional bindings that are particular to e2e users and groups + setup-addon-manifests "addons" "e2e-rbac-bindings" + # Set up manifests of other addons. if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \ [[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \ diff --git a/test/e2e/BUILD b/test/e2e/BUILD index 4e7146d1da8..873b5abd397 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -128,6 +128,7 @@ go_library( "//pkg/apis/extensions:go_default_library", "//pkg/apis/extensions/v1beta1:go_default_library", "//pkg/apis/meta/v1:go_default_library", + "//pkg/apis/rbac/v1alpha1:go_default_library", "//pkg/apis/storage/util:go_default_library", "//pkg/apis/storage/v1beta1:go_default_library", "//pkg/apis/storage/v1beta1/util:go_default_library", @@ -163,6 +164,7 @@ go_library( "//pkg/runtime:go_default_library", "//pkg/runtime/schema:go_default_library", "//pkg/security/apparmor:go_default_library", + "//pkg/serviceaccount:go_default_library", "//pkg/types:go_default_library", "//pkg/util:go_default_library", "//pkg/util/exec:go_default_library", diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index ef5ffa8afff..4a5d27e89e4 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -10,6 +10,7 @@ load( go_library( name = "go_default_library", srcs = [ + "authorizer_util.go", "cleanup.go", "exec_util.go", "federation_util.go", @@ -38,6 +39,7 @@ go_library( "//pkg/api/validation:go_default_library", "//pkg/apimachinery/registered:go_default_library", "//pkg/apis/apps/v1beta1:go_default_library", + "//pkg/apis/authorization/v1beta1:go_default_library", "//pkg/apis/batch:go_default_library", "//pkg/apis/batch/v1:go_default_library", "//pkg/apis/componentconfig:go_default_library", @@ -46,6 +48,7 @@ go_library( "//pkg/apis/meta/v1:go_default_library", "//pkg/apis/meta/v1/unstructured:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", + "//pkg/client/clientset_generated/clientset/typed/authorization/v1beta1:go_default_library", "//pkg/client/clientset_generated/clientset/typed/core/v1:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/conditions:go_default_library", diff --git a/test/e2e/framework/authorizer_util.go b/test/e2e/framework/authorizer_util.go new file mode 100644 index 00000000000..106f5c759ba --- /dev/null +++ b/test/e2e/framework/authorizer_util.go @@ -0,0 +1,58 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "time" + + authorizationv1beta1 "k8s.io/kubernetes/pkg/apis/authorization/v1beta1" + v1beta1authorization "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/authorization/v1beta1" + "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/util/wait" +) + +const ( + policyCachePollInterval = 100 * time.Millisecond + policyCachePollTimeout = 5 * time.Second +) + +// WaitForAuthorizationUpdate checks if the given user can perform the named verb and action. +// If policyCachePollTimeout is reached without the expected condition matching, an error is returned +func WaitForAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviewsGetter, user, namespace, verb string, resource schema.GroupResource, allowed bool) error { + review := &authorizationv1beta1.SubjectAccessReview{ + Spec: authorizationv1beta1.SubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1beta1.ResourceAttributes{ + Group: resource.Group, + Verb: verb, + Resource: resource.Resource, + Namespace: namespace, + }, + User: user, + }, + } + err := wait.Poll(policyCachePollInterval, policyCachePollTimeout, func() (bool, error) { + response, err := c.SubjectAccessReviews().Create(review) + if err != nil { + return false, err + } + if response.Status.Allowed != allowed { + return false, nil + } + return true, nil + }) + return err +} diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index ae8d83dddb3..c64c59eea9b 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -21,6 +21,10 @@ import ( "path/filepath" "time" + legacyv1 "k8s.io/kubernetes/pkg/api/v1" + rbacv1alpha1 "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1" + "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/serviceaccount" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -70,6 +74,32 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() { f.BeforeEach() jig = newTestJig(f.ClientSet) ns = f.Namespace.Name + + // this test wants powerful permissions. Since the namespace names are unique, we can leave this + // lying around so we don't have to race any caches + _, err := jig.client.Rbac().ClusterRoleBindings().Create(&rbacv1alpha1.ClusterRoleBinding{ + ObjectMeta: legacyv1.ObjectMeta{ + Name: f.Namespace.Name + "--cluster-admin", + }, + RoleRef: rbacv1alpha1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "cluster-admin", + }, + Subjects: []rbacv1alpha1.Subject{ + { + Kind: rbacv1alpha1.ServiceAccountKind, + Namespace: f.Namespace.Name, + Name: "default", + }, + }, + }) + framework.ExpectNoError(err) + + err = framework.WaitForAuthorizationUpdate(jig.client.Authorization(), + serviceaccount.MakeUsername(f.Namespace.Name, "default"), + "", "create", schema.GroupResource{Resource: "pods"}, true) + framework.ExpectNoError(err) }) // Before enabling this loadbalancer test in any other test list you must diff --git a/test/e2e/node_problem_detector.go b/test/e2e/node_problem_detector.go index aa8348935cb..b2174ba841d 100644 --- a/test/e2e/node_problem_detector.go +++ b/test/e2e/node_problem_detector.go @@ -24,11 +24,15 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" + legacyv1 "k8s.io/kubernetes/pkg/api/v1" metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" + rbacv1alpha1 "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" coreclientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/serviceaccount" "k8s.io/kubernetes/pkg/util/system" "k8s.io/kubernetes/pkg/util/uuid" "k8s.io/kubernetes/test/e2e/framework" @@ -57,6 +61,32 @@ var _ = framework.KubeDescribe("NodeProblemDetector", func() { configName = "node-problem-detector-config-" + uid // There is no namespace for Node, event recorder will set default namespace for node events. eventNamespace = v1.NamespaceDefault + + // this test wants extra permissions. Since the namespace names are unique, we can leave this + // lying around so we don't have to race any caches + _, err := f.ClientSet.Rbac().ClusterRoleBindings().Create(&rbacv1alpha1.ClusterRoleBinding{ + ObjectMeta: legacyv1.ObjectMeta{ + Name: f.Namespace.Name + "--cluster-admin", + }, + RoleRef: rbacv1alpha1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "cluster-admin", + }, + Subjects: []rbacv1alpha1.Subject{ + { + Kind: rbacv1alpha1.ServiceAccountKind, + Namespace: f.Namespace.Name, + Name: "default", + }, + }, + }) + framework.ExpectNoError(err) + + err = framework.WaitForAuthorizationUpdate(f.ClientSet.Authorization(), + serviceaccount.MakeUsername(f.Namespace.Name, "default"), + "", "create", schema.GroupResource{Resource: "pods"}, true) + framework.ExpectNoError(err) }) // Test kernel monitor. We may add other tests if we have more problem daemons in the future. diff --git a/test/e2e/pre_stop.go b/test/e2e/pre_stop.go index ecfb834367d..a83d022e61e 100644 --- a/test/e2e/pre_stop.go +++ b/test/e2e/pre_stop.go @@ -22,8 +22,12 @@ import ( "time" "k8s.io/kubernetes/pkg/api/v1" + legacyv1 "k8s.io/kubernetes/pkg/api/v1" metav1 "k8s.io/kubernetes/pkg/apis/meta/v1" + rbacv1alpha1 "k8s.io/kubernetes/pkg/apis/rbac/v1alpha1" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/serviceaccount" "k8s.io/kubernetes/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" @@ -163,6 +167,34 @@ func testPreStop(c clientset.Interface, ns string) { var _ = framework.KubeDescribe("PreStop", func() { f := framework.NewDefaultFramework("prestop") + BeforeEach(func() { + // this test wants extra permissions. Since the namespace names are unique, we can leave this + // lying around so we don't have to race any caches + _, err := f.ClientSet.Rbac().ClusterRoleBindings().Create(&rbacv1alpha1.ClusterRoleBinding{ + ObjectMeta: legacyv1.ObjectMeta{ + Name: f.Namespace.Name + "--cluster-admin", + }, + RoleRef: rbacv1alpha1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "cluster-admin", + }, + Subjects: []rbacv1alpha1.Subject{ + { + Kind: rbacv1alpha1.ServiceAccountKind, + Namespace: f.Namespace.Name, + Name: "default", + }, + }, + }) + framework.ExpectNoError(err) + + err = framework.WaitForAuthorizationUpdate(f.ClientSet.Authorization(), + serviceaccount.MakeUsername(f.Namespace.Name, "default"), + "", "create", schema.GroupResource{Resource: "pods"}, true) + framework.ExpectNoError(err) + }) + It("should call prestop when killing a pod [Conformance]", func() { testPreStop(f.ClientSet, f.Namespace.Name) })