From 66ea02684b49dfb2cb647040ecc27008d9a9f203 Mon Sep 17 00:00:00 2001 From: Aravindh Puthiyaparambil Date: Thu, 4 Jan 2024 15:42:19 -0800 Subject: [PATCH] e2e: add NodeLogQuery tests for Windows nodes --- test/e2e/node/kubelet.go | 137 +++++++++++++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 19 deletions(-) diff --git a/test/e2e/node/kubelet.go b/test/e2e/node/kubelet.go index ad4dda9f50f..aa5b4c40af7 100644 --- a/test/e2e/node/kubelet.go +++ b/test/e2e/node/kubelet.go @@ -23,6 +23,7 @@ import ( "io" "os/exec" "path/filepath" + "regexp" "strings" "time" @@ -457,21 +458,32 @@ var _ = SIGDescribe("kubelet", func() { }) // Tests for NodeLogQuery feature - f.Describe("kubectl get --raw \"/api/v1/nodes//proxy/logs/?query=/", feature.NodeLogQuery, "[LinuxOnly]", func() { + f.Describe("kubectl get --raw \"/api/v1/nodes//proxy/logs/?query=/", feature.NodeLogQuery, func() { var linuxNodeName string + var windowsNodeName string ginkgo.BeforeEach(func(ctx context.Context) { - nodes, err := e2enode.GetReadyNodesIncludingTainted(ctx, c) + allNodes, err := e2enode.GetReadyNodesIncludingTainted(ctx, c) framework.ExpectNoError(err) - if len(nodes.Items) == 0 { + if len(allNodes.Items) == 0 { framework.Fail("Expected at least one node to be present") } + // Make a copy of the node list as getLinuxNodes will filter out the Windows nodes + nodes := allNodes.DeepCopy() + linuxNodes := getLinuxNodes(nodes) if len(linuxNodes.Items) == 0 { framework.Fail("Expected at least one Linux node to be present") } + linuxNodeName = linuxNodes.Items[0].Name + + windowsNodes := getWindowsNodes(allNodes) + if len(windowsNodes.Items) == 0 { + framework.Logf("No Windows node found") + } else { + windowsNodeName = windowsNodes.Items[0].Name + } - linuxNodeName = nodes.Items[0].Name }) /* @@ -494,7 +506,7 @@ var _ = SIGDescribe("kubelet", func() { }) /* - Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet" + Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet" returns the kubelet logs */ @@ -509,7 +521,7 @@ var _ = SIGDescribe("kubelet", func() { }) /* - Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet&boot=0" + Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet&boot=0" returns kubelet logs from the current boot */ @@ -524,7 +536,7 @@ var _ = SIGDescribe("kubelet", func() { }) /* - Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet&tailLines=3" + Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet&tailLines=3" returns the last three lines of the kubelet log */ @@ -543,7 +555,7 @@ var _ = SIGDescribe("kubelet", func() { }) /* - Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet&pattern=kubelet" + Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet&pattern=container" returns kubelet logs for the current boot with the pattern container */ @@ -562,8 +574,8 @@ var _ = SIGDescribe("kubelet", func() { }) /* - Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet&sinceTime=" - returns the kubelet since the current date and time. This can be "-- No entries --" which is correct. + Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=kubelet&sinceTime=" + returns the kubelet logs since the current date and time. This can be "-- No entries --" which is correct. */ ginkgo.It("should return the kubelet logs since the current date and time", func() { @@ -581,22 +593,93 @@ var _ = SIGDescribe("kubelet", func() { framework.Failf("Failed to receive the correct kubelet logs or the correct amount of lines of logs") } }) + + /* + Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query="Microsoft-Windows-Security-SPP" + returns the Microsoft-Windows-Security-SPP log + */ + + ginkgo.It("should return the Microsoft-Windows-Security-SPP logs", func(ctx context.Context) { + if len(windowsNodeName) == 0 { + ginkgo.Skip("No Windows node found") + } + ginkgo.By("Starting the command") + tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, ns) + + queryCommand := fmt.Sprintf("/api/v1/nodes/%s/proxy/logs/?query=Microsoft-Windows-Security-SPP", windowsNodeName) + cmd := tk.KubectlCmd("get", "--raw", queryCommand) + result := runKubectlCommand(cmd) + assertContains("ProviderName: Microsoft-Windows-Security-SPP", result) + }) + + /* + Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=Microsoft-Windows-Security-SPP&tailLines=3" + returns the last three lines of the Microsoft-Windows-Security-SPP log + */ + + ginkgo.It("should return the last three lines of the Microsoft-Windows-Security-SPP logs", func(ctx context.Context) { + e2eskipper.SkipUnlessProviderIs(framework.ProvidersWithSSH...) + if len(windowsNodeName) == 0 { + ginkgo.Skip("No Windows node found") + } + ginkgo.By("Starting the command") + tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, ns) + + queryCommand := fmt.Sprintf("/api/v1/nodes/%s/proxy/logs/?query=Microsoft-Windows-Security-SPP&tailLines=3", windowsNodeName) + cmd := tk.KubectlCmd("get", "--raw", queryCommand) + result := runKubectlCommand(cmd) + logs := getWinEventCommandOnNode(windowsNodeName, "Microsoft-Windows-Security-SPP", " -MaxEvents 3") + if trimSpaceNewlineInString(result) != trimSpaceNewlineInString(logs) { + framework.Failf("Failed to receive the correct Microsoft-Windows-Security-SPP logs or the correct amount of lines of logs") + } + }) + + /* + Test if kubectl get --raw "/api/v1/nodes//proxy/logs/?query=Microsoft-Windows-Security-SPP&pattern=Health" + returns the lines of the Microsoft-Windows-Security-SPP log with the pattern Health + */ + + ginkgo.It("should return the Microsoft-Windows-Security-SPP logs with the pattern Health", func(ctx context.Context) { + e2eskipper.SkipUnlessProviderIs(framework.ProvidersWithSSH...) + if len(windowsNodeName) == 0 { + ginkgo.Skip("No Windows node found") + } + ginkgo.By("Starting the command") + tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, ns) + + queryCommand := fmt.Sprintf("/api/v1/nodes/%s/proxy/logs/?query=Microsoft-Windows-Security-SPP&pattern=Health", windowsNodeName) + cmd := tk.KubectlCmd("get", "--raw", queryCommand) + result := runKubectlCommand(cmd) + logs := getWinEventCommandOnNode(windowsNodeName, "Microsoft-Windows-Security-SPP", " | Where-Object -Property Message -Match Health") + if trimSpaceNewlineInString(result) != trimSpaceNewlineInString(logs) { + framework.Failf("Failed to receive the correct Microsoft-Windows-Security-SPP logs or the correct amount of lines of logs") + } + }) }) }) func getLinuxNodes(nodes *v1.NodeList) *v1.NodeList { - e2enode.Filter(nodes, func(node v1.Node) bool { - return isLinuxNode(&node) + filteredNodes := nodes + e2enode.Filter(filteredNodes, func(node v1.Node) bool { + return isNode(&node, "linux") }) - return nodes + return filteredNodes } -func isLinuxNode(node *v1.Node) bool { +func getWindowsNodes(nodes *v1.NodeList) *v1.NodeList { + filteredNodes := nodes + e2enode.Filter(filteredNodes, func(node v1.Node) bool { + return isNode(&node, "windows") + }) + return filteredNodes +} + +func isNode(node *v1.Node, os string) bool { if node == nil { return false } - if os, found := node.Labels[v1.LabelOSStable]; found { - return (os == "linux") + if foundOS, found := node.Labels[v1.LabelOSStable]; found { + return (os == foundOS) } return false } @@ -631,10 +714,26 @@ func assertContains(expectedString string, result string) { framework.Failf("Failed to find \"%s\"", expectedString) } -func journalctlCommandOnNode(nodeName string, args string) string { - result, err := e2essh.NodeExec(context.Background(), nodeName, - "journalctl --utc --no-pager --output=short-precise "+args, framework.TestContext.Provider) +func commandOnNode(nodeName string, cmd string) string { + result, err := e2essh.NodeExec(context.Background(), nodeName, cmd, framework.TestContext.Provider) framework.ExpectNoError(err) e2essh.LogResult(result) return result.Stdout } + +func journalctlCommandOnNode(nodeName string, args string) string { + return commandOnNode(nodeName, "journalctl --utc --no-pager --output=short-precise "+args) +} + +func getWinEventCommandOnNode(nodeName string, providerName, args string) string { + output := commandOnNode(nodeName, "Get-WinEvent -FilterHashtable @{LogName='Application'; ProviderName='"+providerName+"'}"+args+" | Sort-Object TimeCreated | Format-Table -AutoSize -Wrap") + return output +} + +func trimSpaceNewlineInString(s string) string { + // Remove Windows newlines + re := regexp.MustCompile(` +\r?\n +`) + s = re.ReplaceAllString(s, "") + // Replace spaces to account for cases like "\r\n " that could lead to false negatives + return strings.ReplaceAll(s, " ", "") +}