diff --git a/test/e2e/auth/BUILD b/test/e2e/auth/BUILD index 9e52a84464e..bcb7581b130 100644 --- a/test/e2e/auth/BUILD +++ b/test/e2e/auth/BUILD @@ -12,12 +12,14 @@ go_library( "certificates.go", "framework.go", "metadata_concealment.go", + "node_authn.go", "node_authz.go", "pod_security_policy.go", "service_accounts.go", ], importpath = "k8s.io/kubernetes/test/e2e/auth", deps = [ + "//pkg/master/ports:go_default_library", "//pkg/security/apparmor:go_default_library", "//pkg/security/podsecuritypolicy/seccomp:go_default_library", "//pkg/security/podsecuritypolicy/util:go_default_library", diff --git a/test/e2e/auth/node_authn.go b/test/e2e/auth/node_authn.go new file mode 100644 index 00000000000..8e62646ecc5 --- /dev/null +++ b/test/e2e/auth/node_authn.go @@ -0,0 +1,107 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "fmt" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/master/ports" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = SIGDescribe("[Feature:NodeAuthenticator]", func() { + + f := framework.NewDefaultFramework("node-authn") + var ns string + var nodeIPs []string + BeforeEach(func() { + ns = f.Namespace.Name + + nodeList, err := f.ClientSet.CoreV1().Nodes().List(metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeList.Items)).NotTo(BeZero()) + + pickedNode := nodeList.Items[0] + nodeIPs = framework.GetNodeAddresses(&pickedNode, v1.NodeExternalIP) + // The pods running in the cluster can see the internal addresses. + nodeIPs = append(nodeIPs, framework.GetNodeAddresses(&pickedNode, v1.NodeInternalIP)...) + + // make sure ServiceAccount admission controller is enabled, so secret generation on SA creation works + sa, err := f.ClientSet.CoreV1().ServiceAccounts(ns).Get("default", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(sa.Secrets)).NotTo(BeZero()) + }) + + It("The kubelet's main port 10250 should reject requests with no credentials", func() { + pod := createNodeAuthTestPod(f) + for _, nodeIP := range nodeIPs { + // Anonymous authentication is disabled by default + result := framework.RunHostCmdOrDie(ns, pod.Name, fmt.Sprintf("curl -sIk -o /dev/null -w '%s' https://%s:%v/metrics", "%{http_code}", nodeIP, ports.KubeletPort)) + Expect(result).To(Or(Equal("401"), Equal("403")), "the kubelet's main port 10250 should reject requests with no credentials") + } + }) + + It("The kubelet can delegate ServiceAccount tokens to the API server", func() { + By("create a new ServiceAccount for authentication") + trueValue := true + newSA := &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: "node-auth-newSA", + }, + AutomountServiceAccountToken: &trueValue, + } + _, err := f.ClientSet.CoreV1().ServiceAccounts(ns).Create(newSA) + Expect(err).NotTo(HaveOccurred()) + + pod := createNodeAuthTestPod(f) + + for _, nodeIP := range nodeIPs { + result := framework.RunHostCmdOrDie(ns, + pod.Name, + fmt.Sprintf("curl -sIk -o /dev/null -w '%s' --header \"Authorization: Bearer `%s`\" https://%s:%v/metrics", + "%{http_code}", + "cat /var/run/secrets/kubernetes.io/serviceaccount/token", + nodeIP, ports.KubeletPort)) + Expect(result).To(Or(Equal("401"), Equal("403")), "the kubelet can delegate ServiceAccount tokens to the API server") + } + }) +}) + +func createNodeAuthTestPod(f *framework.Framework) *v1.Pod { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-node-authn-", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: "test-node-authn", + Image: imageutils.GetE2EImage(imageutils.Hostexec), + Command: []string{"sleep 3600"}, + }}, + RestartPolicy: v1.RestartPolicyNever, + }, + } + + return f.PodClient().CreateSync(pod) +} diff --git a/test/e2e/framework/kubelet_stats.go b/test/e2e/framework/kubelet_stats.go index 1304cbfcf59..3803f088210 100644 --- a/test/e2e/framework/kubelet_stats.go +++ b/test/e2e/framework/kubelet_stats.go @@ -492,7 +492,7 @@ type usageDataPerContainer struct { } func GetKubeletHeapStats(c clientset.Interface, nodeName string) (string, error) { - client, err := NodeProxyRequest(c, nodeName, "debug/pprof/heap") + client, err := NodeProxyRequest(c, nodeName, "debug/pprof/heap", ports.KubeletPort) if err != nil { return "", err } diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 889c403afec..a4767d0a7eb 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -4492,7 +4492,7 @@ func isElementOf(podUID types.UID, pods *v1.PodList) bool { const proxyTimeout = 2 * time.Minute // NodeProxyRequest performs a get on a node proxy endpoint given the nodename and rest client. -func NodeProxyRequest(c clientset.Interface, node, endpoint string) (restclient.Result, error) { +func NodeProxyRequest(c clientset.Interface, node, endpoint string, port int) (restclient.Result, error) { // proxy tends to hang in some cases when Node is not ready. Add an artificial timeout for this call. // This will leak a goroutine if proxy hangs. #22165 var result restclient.Result @@ -4501,7 +4501,7 @@ func NodeProxyRequest(c clientset.Interface, node, endpoint string) (restclient. result = c.CoreV1().RESTClient().Get(). Resource("nodes"). SubResource("proxy"). - Name(fmt.Sprintf("%v:%v", node, ports.KubeletPort)). + Name(fmt.Sprintf("%v:%v", node, port)). Suffix(endpoint). Do() @@ -4529,7 +4529,7 @@ func GetKubeletRunningPods(c clientset.Interface, node string) (*v1.PodList, err func getKubeletPods(c clientset.Interface, node, resource string) (*v1.PodList, error) { result := &v1.PodList{} - client, err := NodeProxyRequest(c, node, resource) + client, err := NodeProxyRequest(c, node, resource, ports.KubeletPort) if err != nil { return &v1.PodList{}, err } diff --git a/test/e2e/lifecycle/BUILD b/test/e2e/lifecycle/BUILD index 2a004bd7560..4240e223a1c 100644 --- a/test/e2e/lifecycle/BUILD +++ b/test/e2e/lifecycle/BUILD @@ -12,6 +12,7 @@ go_library( "cluster_upgrade.go", "framework.go", "ha_master.go", + "kubelet_security.go", "reboot.go", "resize_nodes.go", "restart.go", @@ -20,6 +21,7 @@ go_library( deps = [ "//pkg/apis/core:go_default_library", "//pkg/kubelet/pod:go_default_library", + "//pkg/master/ports:go_default_library", "//pkg/util/version:go_default_library", "//test/e2e/chaosmonkey:go_default_library", "//test/e2e/common:go_default_library", diff --git a/test/e2e/lifecycle/kubelet_security.go b/test/e2e/lifecycle/kubelet_security.go new file mode 100644 index 00000000000..62040c0b569 --- /dev/null +++ b/test/e2e/lifecycle/kubelet_security.go @@ -0,0 +1,85 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "fmt" + "net" + "net/http" + "time" + + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/master/ports" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = SIGDescribe("Ports Security Check [Feature:KubeletSecurity]", func() { + f := framework.NewDefaultFramework("kubelet-security") + + var node *v1.Node + var nodeName string + + BeforeEach(func() { + nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + Expect(len(nodes.Items)).NotTo(BeZero()) + node = &nodes.Items[0] + nodeName = node.Name + }) + + // make sure kubelet readonly (10255) and cadvisor (4194) ports are disabled via API server proxy + It(fmt.Sprintf("should not be able to proxy to the readonly kubelet port %v using proxy subresource", ports.KubeletReadOnlyPort), func() { + result, err := framework.NodeProxyRequest(f.ClientSet, nodeName, "pods/", ports.KubeletReadOnlyPort) + Expect(err).NotTo(HaveOccurred()) + + var statusCode int + result.StatusCode(&statusCode) + Expect(statusCode).NotTo(Equal(http.StatusOK)) + }) + It("should not be able to proxy to cadvisor port 4194 using proxy subresource", func() { + result, err := framework.NodeProxyRequest(f.ClientSet, nodeName, "containers/", 4194) + Expect(err).NotTo(HaveOccurred()) + + var statusCode int + result.StatusCode(&statusCode) + Expect(statusCode).NotTo(Equal(http.StatusOK)) + }) + + // make sure kubelet readonly (10255) and cadvisor (4194) ports are closed on the public IP address + disabledPorts := []int{ports.KubeletReadOnlyPort, 4194} + for _, port := range disabledPorts { + It(fmt.Sprintf("should not have port %d open on its all public IP addresses", port), func() { + portClosedTest(f, node, port) + }) + } +}) + +// checks whether the target port is closed +func portClosedTest(f *framework.Framework, pickNode *v1.Node, port int) { + nodeAddrs := framework.GetNodeAddresses(pickNode, v1.NodeExternalIP) + Expect(len(nodeAddrs)).NotTo(BeZero()) + + for _, addr := range nodeAddrs { + conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", addr, port), 1*time.Minute) + if err == nil { + conn.Close() + framework.Failf("port %d is not disabled", port) + } + } +}