mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-19 01:40:13 +00:00
1025 lines
34 KiB
Go
1025 lines
34 KiB
Go
/*
|
|
Copyright 2021 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package windows
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/onsi/gomega"
|
|
|
|
semver "github.com/blang/semver/v4"
|
|
"github.com/onsi/ginkgo/v2"
|
|
v1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/fields"
|
|
"k8s.io/apimachinery/pkg/util/uuid"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
clientset "k8s.io/client-go/kubernetes"
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet"
|
|
e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics"
|
|
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
|
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
|
imageutils "k8s.io/kubernetes/test/utils/image"
|
|
admissionapi "k8s.io/pod-security-admission/api"
|
|
)
|
|
|
|
const (
|
|
validation_script = `if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\emptydir)) {
|
|
throw "Cannot find emptydir volume"
|
|
}
|
|
if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\configmap\text.txt)) {
|
|
throw "Cannot find text.txt in configmap-volume"
|
|
}
|
|
$c = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\configmap\text.txt
|
|
if ($c -ne "Lorem ipsum dolor sit amet") {
|
|
throw "Contents of /etc/configmap/text.txt are not as expected"
|
|
}
|
|
if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\hostpath)) {
|
|
throw "Cannot find hostpath volume"
|
|
}
|
|
if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\downwardapi\podname)) {
|
|
throw "Cannot find podname file in downward-api volume"
|
|
}
|
|
$c = Get-Content -Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\downwardapi\podname
|
|
if ($c -ne "host-process-volume-mounts") {
|
|
throw "Contents of /etc/downward-api/podname are not as expected"
|
|
}
|
|
if (-not(Test-Path $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\secret\foo.txt)) {
|
|
throw "Cannot find file foo.txt in secret volume"
|
|
}
|
|
$c = Get-Content $env:CONTAINER_SANDBOX_MOUNT_POINT\etc\secret\foo.txt
|
|
if ($c -ne "bar") {
|
|
Write-Output $c
|
|
throw "Contents of /etc/secret/foo.txt are not as expected"
|
|
}
|
|
# Windows ComputerNames cannot exceed 15 characters, which means that the $env:COMPUTERNAME
|
|
# can only be a substring of $env:NODE_NAME_TEST. We compare it with the hostname instead.
|
|
# The following comparison is case insensitive.
|
|
if ($env:NODE_NAME_TEST -ine "$(hostname)") {
|
|
throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal hostname"
|
|
}
|
|
Write-Output "SUCCESS"`
|
|
)
|
|
|
|
var (
|
|
trueVar = true
|
|
|
|
User_NTAuthorityLocalService = "NT AUTHORITY\\Local Service"
|
|
User_NTAuthoritySystem = "NT AUTHORITY\\SYSTEM"
|
|
)
|
|
|
|
var _ = SIGDescribe("[Feature:WindowsHostProcessContainers] [MinimumKubeletVersion:1.22] HostProcess containers", func() {
|
|
ginkgo.BeforeEach(func() {
|
|
e2eskipper.SkipUnlessNodeOSDistroIs("windows")
|
|
})
|
|
|
|
f := framework.NewDefaultFramework("host-process-test-windows")
|
|
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
|
|
|
|
ginkgo.It("should run as a process on the host/node", func(ctx context.Context) {
|
|
|
|
ginkgo.By("selecting a Windows node")
|
|
targetNode, err := findWindowsNode(ctx, f)
|
|
framework.ExpectNoError(err, "Error finding Windows node")
|
|
framework.Logf("Using node: %v", targetNode.Name)
|
|
|
|
ginkgo.By("scheduling a pod with a container that verifies %COMPUTERNAME% matches selected node name")
|
|
image := imageutils.GetConfig(imageutils.BusyBox)
|
|
podName := "host-process-test-pod"
|
|
// We're passing this to powershell.exe -Command. Inside, we must use apostrophes for strings.
|
|
command := fmt.Sprintf(`& {if ('%s' -ine "$(hostname)") { exit -1 }}`, targetNode.Name)
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthoritySystem,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: image.GetE2EImage(),
|
|
Name: "computer-name-test",
|
|
Command: []string{"powershell.exe", "-Command", command},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeName: targetNode.Name,
|
|
},
|
|
}
|
|
|
|
e2epod.NewPodClient(f).Create(ctx, pod)
|
|
|
|
ginkgo.By("Waiting for pod to run")
|
|
e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
|
|
|
|
ginkgo.By("Then ensuring pod finished running successfully")
|
|
p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
|
|
ctx,
|
|
podName,
|
|
metav1.GetOptions{})
|
|
|
|
framework.ExpectNoError(err, "Error retrieving pod")
|
|
framework.ExpectEqual(p.Status.Phase, v1.PodSucceeded)
|
|
})
|
|
|
|
ginkgo.It("should support init containers", func(ctx context.Context) {
|
|
ginkgo.By("scheduling a pod with a container that verifies init container can configure the node")
|
|
podName := "host-process-init-pods"
|
|
filename := fmt.Sprintf("/testfile%s.txt", string(uuid.NewUUID()))
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthoritySystem,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
InitContainers: []v1.Container{
|
|
{
|
|
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
|
Name: "configure-node",
|
|
Command: []string{"powershell", "-c", "Set-content", "-Path", filename, "-V", "test"},
|
|
},
|
|
},
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
|
Name: "read-configuration",
|
|
Command: []string{"powershell", "-c", "ls", filename},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeSelector: map[string]string{
|
|
"kubernetes.io/os": "windows",
|
|
},
|
|
},
|
|
}
|
|
|
|
e2epod.NewPodClient(f).Create(ctx, pod)
|
|
|
|
ginkgo.By("Waiting for pod to run")
|
|
e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
|
|
|
|
ginkgo.By("Then ensuring pod finished running successfully")
|
|
p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
|
|
ctx,
|
|
podName,
|
|
metav1.GetOptions{})
|
|
|
|
framework.ExpectNoError(err, "Error retrieving pod")
|
|
|
|
if p.Status.Phase != v1.PodSucceeded {
|
|
logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, podName, "read-configuration")
|
|
if err != nil {
|
|
framework.Logf("Error pulling logs: %v", err)
|
|
}
|
|
framework.Logf("Pod phase: %v\nlogs:\n%s", p.Status.Phase, logs)
|
|
}
|
|
framework.ExpectEqual(p.Status.Phase, v1.PodSucceeded)
|
|
})
|
|
|
|
ginkgo.It("container command path validation", func(ctx context.Context) {
|
|
|
|
// The way hostprocess containers are created is being updated in container
|
|
// v1.7 to better support volume mounts and part of these changes include
|
|
// updates how the container's starting process is invoked.
|
|
// These test cases are only valid for containerd v1.6.
|
|
// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-windows/1981-windows-privileged-container-support/README.md
|
|
// for more details.
|
|
ginkgo.By("Ensuring Windows nodes are running containerd v1.6.x")
|
|
windowsNode, err := findWindowsNode(ctx, f)
|
|
framework.ExpectNoError(err, "error finding Windows node")
|
|
r, v, err := getNodeContainerRuntimeAndVersion(windowsNode)
|
|
framework.ExpectNoError(err, "error getting node container runtime and version")
|
|
framework.Logf("Got runtime: %s, version %v, node: %s", r, v, windowsNode.Name)
|
|
|
|
if !strings.EqualFold(r, "containerd") {
|
|
e2eskipper.Skipf("container runtime is not containerd")
|
|
}
|
|
|
|
v1dot7 := semver.MustParse("1.7.0")
|
|
if v.GTE(v1dot7) {
|
|
e2eskipper.Skipf("container runtime is >= 1.7.0")
|
|
}
|
|
|
|
// The following test cases are broken into batches to speed up the test.
|
|
// Each batch will be scheduled as a single pod with a container for each test case.
|
|
// Pods will be scheduled sequentially since the start-up cost of containers is high
|
|
// on Windows and ginkgo may also schedule test cases in parallel.
|
|
tests := [][]struct {
|
|
command []string
|
|
args []string
|
|
workingDir string
|
|
}{
|
|
{
|
|
{
|
|
command: []string{"cmd.exe", "/c", "ver"},
|
|
},
|
|
{
|
|
command: []string{"System32\\cmd.exe", "/c", "ver"},
|
|
workingDir: "c:\\Windows",
|
|
},
|
|
{
|
|
command: []string{"System32\\cmd.exe", "/c", "ver"},
|
|
workingDir: "c:\\Windows\\",
|
|
},
|
|
{
|
|
command: []string{"%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\uname.exe", "-o"},
|
|
},
|
|
},
|
|
{
|
|
{
|
|
command: []string{"%CONTAINER_SANDBOX_MOUNT_POINT%/bin/uname.exe", "-o"},
|
|
},
|
|
{
|
|
command: []string{"%CONTAINER_SANDBOX_MOUNT_POINT%\\bin/uname.exe", "-o"},
|
|
},
|
|
{
|
|
command: []string{"bin/uname.exe", "-o"},
|
|
},
|
|
{
|
|
command: []string{"bin/uname.exe", "-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
|
|
},
|
|
},
|
|
{
|
|
{
|
|
command: []string{"bin\\uname.exe", "-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
|
|
},
|
|
{
|
|
command: []string{"uname.exe", "-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%/bin",
|
|
},
|
|
{
|
|
command: []string{"uname.exe", "-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%/bin/",
|
|
},
|
|
{
|
|
command: []string{"uname.exe", "-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\",
|
|
},
|
|
},
|
|
{
|
|
{
|
|
command: []string{"powershell", "cmd.exe", "/ver"},
|
|
},
|
|
{
|
|
command: []string{"powershell", "c:/Windows/System32/cmd.exe", "/c", "ver"},
|
|
},
|
|
{
|
|
command: []string{"powershell", "c:\\Windows\\System32/cmd.exe", "/c", "ver"},
|
|
},
|
|
{
|
|
command: []string{"powershell", "%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\uname.exe", "-o"},
|
|
},
|
|
},
|
|
{
|
|
{
|
|
command: []string{"powershell", "$env:CONTAINER_SANDBOX_MOUNT_POINT\\bin\\uname.exe", "-o"},
|
|
},
|
|
{
|
|
command: []string{"powershell", "%CONTAINER_SANDBOX_MOUNT_POINT%/bin/uname.exe", "-o"},
|
|
},
|
|
{
|
|
command: []string{"powershell", "$env:CONTAINER_SANDBOX_MOUNT_POINT/bin/uname.exe", "-o"},
|
|
},
|
|
{
|
|
command: []string{"powershell", "bin/uname.exe", "-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
|
|
},
|
|
},
|
|
{
|
|
{
|
|
command: []string{"powershell", "bin/uname.exe", "-o"},
|
|
workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT",
|
|
},
|
|
{
|
|
command: []string{"powershell", "bin\\uname.exe", "-o"},
|
|
workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT",
|
|
},
|
|
{
|
|
command: []string{"powershell", ".\\uname.exe", "-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%/bin",
|
|
},
|
|
{
|
|
command: []string{"powershell", "./uname.exe", "-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%/bin",
|
|
},
|
|
},
|
|
{
|
|
{
|
|
command: []string{"powershell", "./uname.exe", "-o"},
|
|
workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT\\bin\\",
|
|
},
|
|
{
|
|
command: []string{"%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\uname.exe"},
|
|
args: []string{"-o"},
|
|
},
|
|
{
|
|
command: []string{"bin\\uname.exe"},
|
|
args: []string{"-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
|
|
},
|
|
{
|
|
command: []string{"uname.exe"},
|
|
args: []string{"-o"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%\\bin",
|
|
},
|
|
},
|
|
{
|
|
{
|
|
command: []string{"cmd.exe"},
|
|
args: []string{"/c", "dir", "%CONTAINER_SANDBOX_MOUNT_POINT%\\bin\\uname.exe"},
|
|
},
|
|
{
|
|
command: []string{"cmd.exe"},
|
|
args: []string{"/c", "dir", "bin\\uname.exe"},
|
|
workingDir: "%CONTAINER_SANDBOX_MOUNT_POINT%",
|
|
},
|
|
{
|
|
command: []string{"powershell"},
|
|
args: []string{"Get-ChildItem", "-Path", "$env:CONTAINER_SANDBOX_MOUNT_POINT\\bin\\uname.exe"},
|
|
},
|
|
{
|
|
command: []string{"powershell"},
|
|
args: []string{"Get-ChildItem", "-Path", "bin\\uname.exe"},
|
|
workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT",
|
|
},
|
|
},
|
|
{
|
|
{
|
|
command: []string{"powershell"},
|
|
args: []string{"Get-ChildItem", "-Path", "uname.exe"},
|
|
workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/bin",
|
|
},
|
|
},
|
|
}
|
|
|
|
for podIndex, testCaseBatch := range tests {
|
|
image := imageutils.GetConfig(imageutils.BusyBox)
|
|
podName := fmt.Sprintf("host-process-command-%d", podIndex)
|
|
containers := []v1.Container{}
|
|
for containerIndex, testCase := range testCaseBatch {
|
|
containerName := fmt.Sprintf("host-process-command-%d-%d", podIndex, containerIndex)
|
|
ginkgo.By(fmt.Sprintf("Adding a container '%s' to pod '%s' with command: %s, args: %s, workingDir: %s", containerName, podName, strings.Join(testCase.command, " "), strings.Join(testCase.args, " "), testCase.workingDir))
|
|
|
|
container := v1.Container{
|
|
Image: image.GetE2EImage(),
|
|
Name: containerName,
|
|
Command: testCase.command,
|
|
Args: testCase.args,
|
|
WorkingDir: testCase.workingDir,
|
|
}
|
|
containers = append(containers, container)
|
|
}
|
|
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthorityLocalService,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
Containers: containers,
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeSelector: map[string]string{
|
|
"kubernetes.io/os": "windows",
|
|
},
|
|
},
|
|
}
|
|
e2epod.NewPodClient(f).Create(ctx, pod)
|
|
|
|
ginkgo.By(fmt.Sprintf("Waiting for pod '%s' to run", podName))
|
|
e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
|
|
|
|
ginkgo.By("Then ensuring pod finished running successfully")
|
|
p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
|
|
ctx,
|
|
podName,
|
|
metav1.GetOptions{})
|
|
|
|
framework.ExpectNoError(err, "Error retrieving pod")
|
|
|
|
if p.Status.Phase != v1.PodSucceeded {
|
|
framework.Logf("Getting pod events")
|
|
options := metav1.ListOptions{
|
|
FieldSelector: fields.Set{
|
|
"involvedObject.kind": "Pod",
|
|
"involvedObject.name": podName,
|
|
"involvedObject.namespace": f.Namespace.Name,
|
|
}.AsSelector().String(),
|
|
}
|
|
events, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(ctx, options)
|
|
framework.ExpectNoError(err, "Error getting events for failed pod")
|
|
for _, event := range events.Items {
|
|
framework.Logf("%s: %s", event.Reason, event.Message)
|
|
}
|
|
framework.Failf("Pod '%s' did failed.", p.Name)
|
|
}
|
|
}
|
|
|
|
})
|
|
|
|
ginkgo.It("should support various volume mount types", func(ctx context.Context) {
|
|
ns := f.Namespace
|
|
|
|
ginkgo.By("Creating a configmap containing test data and a validation script")
|
|
configMap := &v1.ConfigMap{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: "v1",
|
|
Kind: "ConfigMap",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "sample-config-map",
|
|
},
|
|
Data: map[string]string{
|
|
"text": "Lorem ipsum dolor sit amet",
|
|
"validation-script": validation_script,
|
|
},
|
|
}
|
|
_, err := f.ClientSet.CoreV1().ConfigMaps(ns.Name).Create(ctx, configMap, metav1.CreateOptions{})
|
|
framework.ExpectNoError(err, "unable to create create configmap")
|
|
|
|
ginkgo.By("Creating a secret containing test data")
|
|
secret := &v1.Secret{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: "v1",
|
|
Kind: "Secret",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "sample-secret",
|
|
},
|
|
Type: v1.SecretTypeOpaque,
|
|
Data: map[string][]byte{
|
|
"foo": []byte("bar"),
|
|
},
|
|
}
|
|
_, err = f.ClientSet.CoreV1().Secrets(ns.Name).Create(ctx, secret, metav1.CreateOptions{})
|
|
framework.ExpectNoError(err, "unable to create secret")
|
|
|
|
ginkgo.By("Creating a pod with a HostProcess container that uses various types of volume mounts")
|
|
|
|
podAndContainerName := "host-process-volume-mounts"
|
|
pod := makeTestPodWithVolumeMounts(podAndContainerName)
|
|
|
|
e2epod.NewPodClient(f).Create(ctx, pod)
|
|
|
|
ginkgo.By("Waiting for pod to run")
|
|
e2epod.NewPodClient(f).WaitForFinish(ctx, podAndContainerName, 3*time.Minute)
|
|
|
|
logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, ns.Name, podAndContainerName, podAndContainerName)
|
|
framework.ExpectNoError(err, "Error getting pod logs")
|
|
framework.Logf("Container logs: %s", logs)
|
|
|
|
ginkgo.By("Then ensuring pod finished running successfully")
|
|
p, err := f.ClientSet.CoreV1().Pods(ns.Name).Get(
|
|
ctx,
|
|
podAndContainerName,
|
|
metav1.GetOptions{})
|
|
|
|
framework.ExpectNoError(err, "Error retrieving pod")
|
|
framework.ExpectEqual(p.Status.Phase, v1.PodSucceeded)
|
|
})
|
|
|
|
ginkgo.It("metrics should report count of started and failed to start HostProcess containers", func(ctx context.Context) {
|
|
ginkgo.By("Selecting a Windows node")
|
|
targetNode, err := findWindowsNode(ctx, f)
|
|
framework.ExpectNoError(err, "Error finding Windows node")
|
|
framework.Logf("Using node: %v", targetNode.Name)
|
|
|
|
ginkgo.By("Getting initial kubelet metrics values")
|
|
beforeMetrics, err := getCurrentHostProcessMetrics(ctx, f, targetNode.Name)
|
|
framework.ExpectNoError(err, "Error getting initial kubelet metrics for node")
|
|
framework.Logf("Initial HostProcess container metrics -- StartedContainers: %v, StartedContainersErrors: %v, StartedInitContainers: %v, StartedInitContainersErrors: %v",
|
|
beforeMetrics.StartedContainersCount, beforeMetrics.StartedContainersErrorCount, beforeMetrics.StartedInitContainersCount, beforeMetrics.StartedInitContainersErrorCount)
|
|
|
|
ginkgo.By("Scheduling a pod with a HostProcess init container that will fail")
|
|
|
|
badUserName := "bad-user-name"
|
|
|
|
podName := "host-process-metrics-pod-failing-init-container"
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthoritySystem,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
InitContainers: []v1.Container{
|
|
{
|
|
SecurityContext: &v1.SecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
RunAsUserName: &badUserName,
|
|
},
|
|
},
|
|
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
|
Name: "failing-init-container",
|
|
Command: []string{"foobar.exe"},
|
|
},
|
|
},
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
|
Name: "container",
|
|
Command: []string{"cmd.exe", "/c", "exit", "/b", "0"},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeName: targetNode.Name,
|
|
},
|
|
}
|
|
|
|
e2epod.NewPodClient(f).Create(ctx, pod)
|
|
e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
|
|
|
|
ginkgo.By("Scheduling a pod with a HostProcess container that will fail")
|
|
podName = "host-process-metrics-pod-failing-container"
|
|
pod = &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthoritySystem,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
Containers: []v1.Container{
|
|
{
|
|
SecurityContext: &v1.SecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
RunAsUserName: &badUserName,
|
|
},
|
|
},
|
|
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
|
Name: "failing-container",
|
|
Command: []string{"foobar.exe"},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeName: targetNode.Name,
|
|
},
|
|
}
|
|
|
|
e2epod.NewPodClient(f).Create(ctx, pod)
|
|
e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
|
|
|
|
ginkgo.By("Getting subsequent kubelet metrics values")
|
|
|
|
afterMetrics, err := getCurrentHostProcessMetrics(ctx, f, targetNode.Name)
|
|
framework.ExpectNoError(err, "Error getting subsequent kubelet metrics for node")
|
|
framework.Logf("Subsequent HostProcess container metrics -- StartedContainers: %v, StartedContainersErrors: %v, StartedInitContainers: %v, StartedInitContainersErrors: %v",
|
|
afterMetrics.StartedContainersCount, afterMetrics.StartedContainersErrorCount, afterMetrics.StartedInitContainersCount, afterMetrics.StartedInitContainersErrorCount)
|
|
|
|
// Note: This test performs relative comparisons to ensure metrics values were logged and does not validate specific values.
|
|
// This done so the test can be run in parallel with other tests which may start HostProcess containers on the same node.
|
|
ginkgo.By("Ensuring metrics were updated")
|
|
gomega.Expect(beforeMetrics.StartedContainersCount).To(gomega.BeNumerically("<", afterMetrics.StartedContainersCount), "Count of started HostProcess containers should increase")
|
|
gomega.Expect(beforeMetrics.StartedContainersErrorCount).To(gomega.BeNumerically("<", afterMetrics.StartedContainersErrorCount), "Count of started HostProcess errors containers should increase")
|
|
gomega.Expect(beforeMetrics.StartedInitContainersCount).To(gomega.BeNumerically("<", afterMetrics.StartedInitContainersCount), "Count of started HostProcess init containers should increase")
|
|
gomega.Expect(beforeMetrics.StartedInitContainersErrorCount).To(gomega.BeNumerically("<", afterMetrics.StartedInitContainersErrorCount), "Count of started HostProcess errors init containers should increase")
|
|
})
|
|
|
|
ginkgo.It("container stats validation", func(ctx context.Context) {
|
|
ginkgo.By("selecting a Windows node")
|
|
targetNode, err := findWindowsNode(ctx, f)
|
|
framework.ExpectNoError(err, "Error finding Windows node")
|
|
framework.Logf("Using node: %v", targetNode.Name)
|
|
|
|
ginkgo.By("schedule a pod with a HostProcess container")
|
|
image := imageutils.GetConfig(imageutils.BusyBox)
|
|
podName := "host-process-stats-pod"
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthorityLocalService,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: image.GetE2EImage(),
|
|
Name: "host-process-stats",
|
|
Command: []string{"powershell.exe", "-Command", "Write-Host 'Hello'; sleep -Seconds 600"},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeName: targetNode.Name,
|
|
},
|
|
}
|
|
|
|
e2epod.NewPodClient(f).Create(ctx, pod)
|
|
|
|
ginkgo.By("Waiting for the pod to start running")
|
|
timeout := 3 * time.Minute
|
|
e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 1, 0, timeout)
|
|
|
|
ginkgo.By("Getting container stats for pod")
|
|
statsChecked := false
|
|
|
|
wait.Poll(2*time.Second, timeout, func() (bool, error) {
|
|
return ensurePodsStatsOnNode(ctx, f.ClientSet, f.Namespace.Name, targetNode.Name, &statsChecked)
|
|
})
|
|
|
|
if !statsChecked {
|
|
framework.Failf("Failed to get stats for pod %s/%s", f.Namespace.Name, podName)
|
|
}
|
|
})
|
|
|
|
ginkgo.It("should support querying api-server using in-cluster config", func(ctx context.Context) {
|
|
// This functionality is only support on containerd v1.7+
|
|
skipUnlessContainerdOneSevenOrGreater(ctx, f)
|
|
|
|
ginkgo.By("Scheduling a pod that runs agnhost inclusterclient")
|
|
podName := "host-process-agnhost-icc"
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthoritySystem,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: imageutils.GetE2EImage(imageutils.Agnhost),
|
|
Name: "hpc-agnhost",
|
|
// TODO: Figure out why we need to copy agnhost as agnhost.exe here
|
|
// Possibly related to https://github.com/microsoft/hcsshim/issues/1128 and
|
|
// https://github.com/microsoft/hcsshim/pull/1174 updating PATHEXT here doesn't
|
|
// seem address the issue.
|
|
Command: []string{"cmd", "/C", "copy", "c:\\hpc\\agnhost", "c:\\hpc\\agnhost.exe", "&&", "c:\\hpc\\agnhost.exe", "inclusterclient"},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeSelector: map[string]string{
|
|
"kubernetes.io/os": "windows",
|
|
},
|
|
},
|
|
}
|
|
|
|
pc := e2epod.NewPodClient(f)
|
|
pc.Create(ctx, pod)
|
|
|
|
ginkgo.By("Waiting for pod to run")
|
|
e2epod.WaitForPodsRunningReady(ctx, f.ClientSet, f.Namespace.Name, 1, 0, 3*time.Minute)
|
|
|
|
ginkgo.By("Waiting for 60 seconds")
|
|
// We wait an additional 60 seconds after the pod is Running because the
|
|
// `InClusterClient` app in the agnhost image waits 30 seconds before
|
|
// starting to make queries - https://github.com/kubernetes/kubernetes/blob/ceee3783ed963497db669504241af2ca6a11610f/test/images/agnhost/inclusterclient/main.go#L51
|
|
time.Sleep(60 * time.Second)
|
|
|
|
ginkgo.By("Ensuring the test app was able to successfully query the api-server")
|
|
logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, podName, "hpc-agnhost")
|
|
framework.ExpectNoError(err, "Error getting pod logs")
|
|
|
|
framework.Logf("Logs: %s\n", logs)
|
|
|
|
framework.ExpectEqual(
|
|
strings.Contains(logs, "calling /healthz"),
|
|
true,
|
|
"app logs should contain 'calling /healthz'")
|
|
|
|
framework.ExpectEqual(
|
|
strings.Contains(logs, "status=failed"),
|
|
false,
|
|
"app logs should not contain 'status=failed")
|
|
})
|
|
|
|
ginkgo.It("should run as localgroup accounts", func(ctx context.Context) {
|
|
// This functionality is only supported on containerd v1.7+
|
|
skipUnlessContainerdOneSevenOrGreater(ctx, f)
|
|
|
|
ginkgo.By("Scheduling a pod that creates a localgroup from an init container then starts a container using that group")
|
|
localGroupName := getRandomUserGrounName()
|
|
podName := "host-process-localgroup-pod"
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthoritySystem,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
InitContainers: []v1.Container{
|
|
{
|
|
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
|
Name: "setup",
|
|
Command: []string{"cmd", "/C", "net", "localgroup", localGroupName, "/add"},
|
|
},
|
|
},
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
|
Name: "localgroup-container",
|
|
Command: []string{"cmd", "/C", "whoami"},
|
|
SecurityContext: &v1.SecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
RunAsUserName: &localGroupName,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeSelector: map[string]string{
|
|
"kubernetes.io/os": "windows",
|
|
},
|
|
},
|
|
}
|
|
|
|
e2epod.NewPodClient(f).Create(ctx, pod)
|
|
|
|
ginkgo.By("Waiting for pod to run")
|
|
e2epod.NewPodClient(f).WaitForFinish(ctx, podName, 3*time.Minute)
|
|
|
|
ginkgo.By("Then ensuring pod finished running successfully")
|
|
p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(
|
|
context.TODO(),
|
|
podName,
|
|
metav1.GetOptions{})
|
|
|
|
framework.ExpectNoError(err, "error retrieving pod")
|
|
framework.ExpectEqual(p.Status.Phase, v1.PodSucceeded)
|
|
|
|
// whoami will output %COMPUTER_NAME%/{randomly generated username} here.
|
|
// It is sufficient to just check that the logs do not container `nt authority`
|
|
// because all of the 'built-in' accounts that can be used with HostProcess
|
|
// are prefixed with this.
|
|
ginkgo.By("Then ensuring pod was not running as a system account")
|
|
logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, podName, "localgroup-container")
|
|
framework.ExpectNoError(err, "error retrieving container logs")
|
|
framework.Logf("Pod logs: %s", logs)
|
|
framework.ExpectEqual(
|
|
strings.Contains(
|
|
strings.ToLower(logs),
|
|
"nt authority"),
|
|
false,
|
|
"Container runs 'whoami' and logs should not contain 'nt authority'")
|
|
})
|
|
|
|
})
|
|
|
|
func makeTestPodWithVolumeMounts(name string) *v1.Pod {
|
|
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
|
|
return &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
SecurityContext: &v1.PodSecurityContext{
|
|
WindowsOptions: &v1.WindowsSecurityContextOptions{
|
|
HostProcess: &trueVar,
|
|
RunAsUserName: &User_NTAuthoritySystem,
|
|
},
|
|
},
|
|
HostNetwork: true,
|
|
Containers: []v1.Container{
|
|
{
|
|
Image: imageutils.GetE2EImage(imageutils.BusyBox),
|
|
Name: name,
|
|
Command: []string{"powershell.exe", "./etc/configmap/validationscript.ps1"},
|
|
Env: []v1.EnvVar{
|
|
{
|
|
Name: "NODE_NAME_TEST",
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "spec.nodeName",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: "emptydir-volume",
|
|
MountPath: "/etc/emptydir",
|
|
},
|
|
{
|
|
Name: "configmap-volume",
|
|
MountPath: "/etc/configmap",
|
|
},
|
|
{
|
|
Name: "hostpath-volume",
|
|
MountPath: "/etc/hostpath",
|
|
},
|
|
{
|
|
Name: "downwardapi-volume",
|
|
MountPath: "/etc/downwardapi",
|
|
},
|
|
{
|
|
Name: "secret-volume",
|
|
MountPath: "/etc/secret",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
NodeSelector: map[string]string{
|
|
"kubernetes.io/os": "windows",
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: "emptydir-volume",
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{
|
|
Medium: v1.StorageMediumDefault,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "configmap-volume",
|
|
VolumeSource: v1.VolumeSource{
|
|
ConfigMap: &v1.ConfigMapVolumeSource{
|
|
LocalObjectReference: v1.LocalObjectReference{
|
|
Name: "sample-config-map",
|
|
},
|
|
Items: []v1.KeyToPath{
|
|
{
|
|
Key: "text",
|
|
Path: "text.txt",
|
|
},
|
|
{
|
|
Key: "validation-script",
|
|
Path: "validationscript.ps1",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "hostpath-volume",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/var/hostpath",
|
|
Type: &hostPathDirectoryOrCreate,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "downwardapi-volume",
|
|
VolumeSource: v1.VolumeSource{
|
|
DownwardAPI: &v1.DownwardAPIVolumeSource{
|
|
Items: []v1.DownwardAPIVolumeFile{
|
|
{
|
|
Path: "podname",
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "metadata.name",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "secret-volume",
|
|
VolumeSource: v1.VolumeSource{
|
|
Secret: &v1.SecretVolumeSource{
|
|
SecretName: "sample-secret",
|
|
Items: []v1.KeyToPath{
|
|
{
|
|
Key: "foo",
|
|
Path: "foo.txt",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
type HostProcessContainersMetrics struct {
|
|
StartedContainersCount int64
|
|
StartedContainersErrorCount int64
|
|
StartedInitContainersCount int64
|
|
StartedInitContainersErrorCount int64
|
|
}
|
|
|
|
// ensurePodsStatsOnNode returns a boolean after checking Pods statistics on a particular node
|
|
func ensurePodsStatsOnNode(ctx context.Context, c clientset.Interface, namespace, nodeName string, statsChecked *bool) (bool, error) {
|
|
nodeStats, err := e2ekubelet.GetStatsSummary(ctx, c, nodeName)
|
|
framework.ExpectNoError(err, "Error getting node stats")
|
|
|
|
for _, podStats := range nodeStats.Pods {
|
|
if podStats.PodRef.Namespace != namespace {
|
|
continue
|
|
}
|
|
|
|
// check various pod stats
|
|
if *podStats.CPU.UsageCoreNanoSeconds <= 0 {
|
|
framework.Logf("Pod %s/%s stats report cpu usage equal to %v but should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds)
|
|
return false, nil
|
|
}
|
|
if *podStats.Memory.WorkingSetBytes <= 0 {
|
|
framework.Logf("Pod %s/%s stats report memory usage equal to %v but should be greater than 0", podStats.PodRef.Namespace, podStats.PodRef.Name, *podStats.CPU.UsageCoreNanoSeconds)
|
|
return false, nil
|
|
}
|
|
|
|
for _, containerStats := range podStats.Containers {
|
|
*statsChecked = true
|
|
|
|
// check various container stats
|
|
if *containerStats.CPU.UsageCoreNanoSeconds <= 0 {
|
|
framework.Logf("Container %s stats report cpu usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.CPU.UsageCoreNanoSeconds)
|
|
return false, nil
|
|
}
|
|
if *containerStats.Memory.WorkingSetBytes <= 0 {
|
|
framework.Logf("Container %s stats report memory usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.Memory.WorkingSetBytes)
|
|
return false, nil
|
|
}
|
|
if *containerStats.Logs.UsedBytes <= 0 {
|
|
framework.Logf("Container %s stats log usage equal to %v but should be greater than 0", containerStats.Name, *containerStats.Logs.UsedBytes)
|
|
return false, nil
|
|
}
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// getCurrentHostProcessMetrics returns a HostPRocessContainersMetrics object. Any metrics that do not have any
|
|
// values reported will be set to 0.
|
|
func getCurrentHostProcessMetrics(ctx context.Context, f *framework.Framework, nodeName string) (HostProcessContainersMetrics, error) {
|
|
var result HostProcessContainersMetrics
|
|
|
|
metrics, err := e2emetrics.GetKubeletMetrics(ctx, f.ClientSet, nodeName)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
|
|
for _, sample := range metrics["started_host_process_containers_total"] {
|
|
switch sample.Metric["container_type"] {
|
|
case "container":
|
|
result.StartedContainersCount = int64(sample.Value)
|
|
case "init_container":
|
|
result.StartedInitContainersCount = int64(sample.Value)
|
|
}
|
|
}
|
|
|
|
// note: accumulate failures of all types (ErrImagePull, RunContainerError, etc)
|
|
// for each container type here.
|
|
for _, sample := range metrics["started_host_process_containers_errors_total"] {
|
|
switch sample.Metric["container_type"] {
|
|
case "container":
|
|
result.StartedContainersErrorCount += int64(sample.Value)
|
|
case "init_container":
|
|
result.StartedInitContainersErrorCount += int64(sample.Value)
|
|
}
|
|
}
|
|
|
|
return result, nil
|
|
}
|