From 46c820e793a353f307461dc94e6d4b81885b772e Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Thu, 13 Feb 2020 15:23:50 +0000 Subject: [PATCH] test images: uses nanoserver Using Windows nanoserver container images as a base instead of the current Windows servercore image will reduce the image size by about ~10x. However, the nanoserver image lacks several things we need: - netapi32.dll - powershell - certain powershell commands - chocolatey cannot be used When building the nanoserver images, we are going to use a Windows servercore helper, in which we are going to install the necessary dependencies, and then copy them over to our nanoserver image, including necessary DLLs. Other notable changes include: - switch from wget to curl (wget was a powershell alias). - implement in code getting the DNS suffix list and DNS server list. - reimplement getting file permissions for mounttest. --- build/dependencies.yaml | 2 +- test/images/agnhost/Dockerfile_windows | 20 ++- test/images/agnhost/VERSION | 2 +- test/images/agnhost/agnhost.go | 2 +- test/images/agnhost/dns/BUILD | 8 +- test/images/agnhost/dns/dns_windows.go | 143 +++++++++++++++--- .../agnhost/mounttest/filePermissions.ps1 | 46 ++---- test/images/busybox/BASEIMAGE | 6 +- test/images/busybox/Dockerfile_windows | 49 +++++- test/images/image-util.sh | 4 +- .../windows-image-builder-helper/BASEIMAGE | 3 + .../Dockerfile_windows | 51 +++++++ .../windows-image-builder-helper/VERSION | 1 + 13 files changed, 257 insertions(+), 80 deletions(-) create mode 100644 test/images/windows-image-builder-helper/BASEIMAGE create mode 100644 test/images/windows-image-builder-helper/Dockerfile_windows create mode 100644 test/images/windows-image-builder-helper/VERSION diff --git a/build/dependencies.yaml b/build/dependencies.yaml index b6c03444017..9b1daa22ffa 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -1,7 +1,7 @@ dependencies: # agnhost: bump this one first - name: "agnhost" - version: "2.22" + version: "2.23" refPaths: - path: test/images/agnhost/VERSION match: \d.\d diff --git a/test/images/agnhost/Dockerfile_windows b/test/images/agnhost/Dockerfile_windows index 76edbdbb510..4572e5c17bb 100644 --- a/test/images/agnhost/Dockerfile_windows +++ b/test/images/agnhost/Dockerfile_windows @@ -13,6 +13,8 @@ # limitations under the License. ARG BASEIMAGE +ARG REGISTRY +FROM $REGISTRY/windows-image-builder-helper:1.0 as helper FROM $BASEIMAGE # from dnsutils image @@ -24,18 +26,17 @@ FROM $BASEIMAGE # - curl, nc: used by a lot of e2e tests (inherited from BASEIMAGE) # from iperf image # install necessary packages: iperf -ENV chocolateyUseWindowsCompression false +COPY --from=helper /dig /dig +COPY --from=helper /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll + +RUN setx /M PATH "C:\dig\;%PATH% RUN powershell -Command "\ - iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \ - choco feature disable --name showDownloadProgress; \ - choco install bind-toolsonly --version 9.10.3 -y -RUN powershell -Command "\ - wget -uri 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -OutFile C:\coredns.tgz;\ - tar -xzvf C:\coredns.tgz;\ + curl.exe -L 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -o C:\coredns.tgz;\ + tar.exe -xzvf C:\coredns.tgz;\ Remove-Item C:\coredns.tgz" RUN powershell -Command "\ - wget -uri 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -OutFile C:\iperf.zip;\ + curl.exe -L 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -o C:\iperf.zip;\ Expand-Archive -Path C:\iperf.zip -DestinationPath C:\ -Force;\ Rename-Item C:\iperf-2.0.9-win64 C:\iperf;\ Remove-Item C:\iperf.zip" @@ -53,6 +54,9 @@ RUN mkdir C:\uploads ADD porter/localhost.crt localhost.crt ADD porter/localhost.key localhost.key +# from mounttest +ADD mounttest/filePermissions.ps1 filePermissions.ps1 + ADD agnhost agnhost # needed for the entrypoint-tester related tests. Some of the entrypoint-tester related tests diff --git a/test/images/agnhost/VERSION b/test/images/agnhost/VERSION index 4699fb07e80..aac433c1a6b 100644 --- a/test/images/agnhost/VERSION +++ b/test/images/agnhost/VERSION @@ -1 +1 @@ -2.22 +2.23 diff --git a/test/images/agnhost/agnhost.go b/test/images/agnhost/agnhost.go index 6e899f76e28..a2a718f1c59 100644 --- a/test/images/agnhost/agnhost.go +++ b/test/images/agnhost/agnhost.go @@ -51,7 +51,7 @@ import ( func main() { rootCmd := &cobra.Command{ Use: "app", - Version: "2.22", + Version: "2.23", } rootCmd.AddCommand(auditproxy.CmdAuditProxy) diff --git a/test/images/agnhost/dns/BUILD b/test/images/agnhost/dns/BUILD index 193173737f1..e4b904450f2 100644 --- a/test/images/agnhost/dns/BUILD +++ b/test/images/agnhost/dns/BUILD @@ -15,7 +15,13 @@ go_library( importpath = "k8s.io/kubernetes/test/images/agnhost/dns", deps = [ "//vendor/github.com/spf13/cobra:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/golang.org/x/sys/windows:go_default_library", + "//vendor/golang.org/x/sys/windows/registry:go_default_library", + ], + "//conditions:default": [], + }), ) filegroup( diff --git a/test/images/agnhost/dns/dns_windows.go b/test/images/agnhost/dns/dns_windows.go index 191958c6534..bd408c99ccb 100644 --- a/test/images/agnhost/dns/dns_windows.go +++ b/test/images/agnhost/dns/dns_windows.go @@ -1,3 +1,5 @@ +// +build windows + /* Copyright 2019 The Kubernetes Authors. @@ -17,40 +19,131 @@ limitations under the License. package dns import ( - "bytes" - "os/exec" + "fmt" "strings" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/registry" ) -const etcHostsFile = "C:/Windows/System32/drivers/etc/hosts" +const ( + etcHostsFile = "C:/Windows/System32/drivers/etc/hosts" + netRegistry = `System\CurrentControlSet\Services\TCPIP\Parameters` + netIfacesRegistry = `System\CurrentControlSet\Services\TCPIP\Parameters\Interfaces` + maxHostnameLen = 128 + maxDomainNameLen = 128 + maxScopeIDLen = 256 +) +// FixedInfo information: https://docs.microsoft.com/en-us/windows/win32/api/iptypes/ns-iptypes-fixed_info_w2ksp1 +type FixedInfo struct { + HostName [maxHostnameLen + 4]byte + DomainName [maxDomainNameLen + 4]byte + CurrentDNSServer *syscall.IpAddrString + DNSServerList syscall.IpAddrString + NodeType uint32 + ScopeID [maxScopeIDLen + 4]byte + EnableRouting uint32 + EnableProxy uint32 + EnableDNS uint32 +} + +var ( + // GetNetworkParams can be found in iphlpapi.dll + // see: https://docs.microsoft.com/en-us/windows/win32/api/iphlpapi/nf-iphlpapi-getnetworkparams?redirectedfrom=MSDN + iphlpapidll = windows.MustLoadDLL("iphlpapi.dll") + procGetNetworkParams = iphlpapidll.MustFindProc("GetNetworkParams") +) + +func elemInList(elem string, list []string) bool { + for _, e := range list { + if e == elem { + return true + } + } + return false +} + +func getRegistryValue(reg, key string) string { + regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, reg, registry.QUERY_VALUE) + if err != nil { + return "" + } + defer regKey.Close() + + regValue, _, err := regKey.GetStringValue(key) + if err != nil { + return "" + } + return regValue +} + +// getDNSSuffixList reads DNS config file and returns the list of configured DNS suffixes func getDNSSuffixList() []string { - output := runCommand("powershell", "-Command", "(Get-DnsClient)[0].SuffixSearchList") - if len(output) > 0 { - return strings.Split(output, "\r\n") + // We start with the general suffix list that apply to all network connections. + allSuffixes := []string{} + suffixes := getRegistryValue(netRegistry, "SearchList") + if suffixes != "" { + allSuffixes = strings.Split(suffixes, ",") } - panic("Could not find DNS search list!") -} - -func getDNSServerList() []string { - output := runCommand("powershell", "-Command", "(Get-DnsClientServerAddress).ServerAddresses") - if len(output) > 0 { - return strings.Split(output, "\r\n") - } - - panic("Could not find DNS Server list!") -} - -func runCommand(name string, arg ...string) string { - var out bytes.Buffer - cmd := exec.Command(name, arg...) - cmd.Stdout = &out - - err := cmd.Run() + // Then we append the network-specific DNS suffix lists. + regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, netIfacesRegistry, registry.ENUMERATE_SUB_KEYS) if err != nil { panic(err) } + defer regKey.Close() - return strings.TrimSpace(out.String()) + ifaces, err := regKey.ReadSubKeyNames(0) + if err != nil { + panic(err) + } + for _, iface := range ifaces { + suffixes := getRegistryValue(fmt.Sprintf("%s\\%s", netIfacesRegistry, iface), "SearchList") + if suffixes == "" { + continue + } + for _, suffix := range strings.Split(suffixes, ",") { + if !elemInList(suffix, allSuffixes) { + allSuffixes = append(allSuffixes, suffix) + } + } + } + + return allSuffixes +} + +func getNetworkParams() *FixedInfo { + // We don't know how big we should make the byte buffer, but the call will tell us by + // setting the size afterwards. + var size int + buffer := make([]byte, 1) + procGetNetworkParams.Call( + uintptr(unsafe.Pointer(&buffer[0])), + uintptr(unsafe.Pointer(&size)), + ) + + buffer = make([]byte, size) + procGetNetworkParams.Call( + uintptr(unsafe.Pointer(&buffer[0])), + uintptr(unsafe.Pointer(&size)), + ) + + info := (*FixedInfo)(unsafe.Pointer(&buffer[0])) + return info +} + +func getDNSServerList() []string { + dnsServerList := []string{} + fixedInfo := getNetworkParams() + list := &(fixedInfo.DNSServerList) + + for list != nil { + dnsServer := strings.TrimRight(string(list.IpAddress.String[:]), "\x00") + dnsServerList = append(dnsServerList, dnsServer) + list = list.Next + } + return dnsServerList } diff --git a/test/images/agnhost/mounttest/filePermissions.ps1 b/test/images/agnhost/mounttest/filePermissions.ps1 index 57ef50aa921..b474acd2e0e 100644 --- a/test/images/agnhost/mounttest/filePermissions.ps1 +++ b/test/images/agnhost/mounttest/filePermissions.ps1 @@ -28,59 +28,41 @@ $EXECUTE_PERMISSIONS = 0x0001 -bor 0x0020 function GetFilePermissions($path) { - $objPath = "Win32_LogicalFileSecuritySetting='$path'" - $output = Invoke-WmiMethod -Namespace root/cimv2 -Path $objPath -Name GetSecurityDescriptor - - if ($output.ReturnValue -ne 0) { - $retVal = $output.ReturnValue - Write-Error "GetSecurityDescriptor invocation failed with code: $retVal" - exit 1 - } - - $fileSD = $output.Descriptor - $fileOwnerGroup = $fileSD.Group - $fileOwner = $fileSD.Owner - - if ($fileOwnerGroup.Name -eq $null -and $fileOwnerGroup.Domain -eq $null) { - # the file owner's group is not recognized. Check if the Owner itself is - # a group, and if so, default the group to it. - net user $fileOwner.Name > $null 2> $null - if (-not $?) { - $fileOwnerGroup = $fileOwner - } - - } + $fileAcl = Get-Acl -Path $path + $fileOwner = $fileAcl.Owner + $fileGroup = $fileAcl.Group $userMask = 0 $groupMask = 0 $otherMask = 0 - foreach ($ace in $fileSD.DACL) { - $mask = 0 - if ($ace.AceType -ne 0) { - # not an Allow ACE, skip. + foreach ($rule in $fileAcl.Access) { + if ($rule.AccessControlType -ne [Security.AccessControl.AccessControlType]::Allow) { + # not an allow rule, skipping. continue } + $mask = 0 + $rights = $rule.FileSystemRights.value__ # convert mask. - if ( ($ace.AccessMask -band $READ_PERMISSIONS) -eq $READ_PERMISSIONS ) { + if ( ($rights -band $READ_PERMISSIONS) -eq $READ_PERMISSIONS ) { $mask = $mask -bor 4 } - if ( ($ace.AccessMask -band $WRITE_PERMISSIONS) -eq $WRITE_PERMISSIONS ) { + if ( ($rights -band $WRITE_PERMISSIONS) -eq $WRITE_PERMISSIONS ) { $mask = $mask -bor 2 } - if ( ($ace.AccessMask -band $EXECUTE_PERMISSIONS) -eq $EXECUTE_PERMISSIONS ) { + if ( ($rights -band $EXECUTE_PERMISSIONS) -eq $EXECUTE_PERMISSIONS ) { $mask = $mask -bor 1 } # detect mask type. - if ($ace.Trustee.Equals($fileOwner)) { + if ($rule.IdentityReference.Value.Equals($fileOwner)) { $userMask = $mask } - if ($ace.Trustee.Equals($fileOwnerGroup)) { + if ($rule.IdentityReference.Value.Equals($fileGroup)) { $groupMask = $mask } - if ($ace.Trustee.Name.ToLower() -eq "users") { + if ($rule.IdentityReference.Value.ToLower().Contains("users")) { $otherMask = $mask } } diff --git a/test/images/busybox/BASEIMAGE b/test/images/busybox/BASEIMAGE index ce71d103b81..25a8a313cad 100644 --- a/test/images/busybox/BASEIMAGE +++ b/test/images/busybox/BASEIMAGE @@ -1,3 +1,3 @@ -windows/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019 -windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903 -windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909 +windows/amd64/1809=mcr.microsoft.com/windows/nanoserver:1809 +windows/amd64/1903=mcr.microsoft.com/windows/nanoserver:1903 +windows/amd64/1909=mcr.microsoft.com/windows/nanoserver:1909 diff --git a/test/images/busybox/Dockerfile_windows b/test/images/busybox/Dockerfile_windows index c4a508fa96a..be60037e0c2 100644 --- a/test/images/busybox/Dockerfile_windows +++ b/test/images/busybox/Dockerfile_windows @@ -15,20 +15,57 @@ ARG BASEIMAGE from $BASEIMAGE as prep -ENV CURL_VERSION 7.57.0 +ENV CURL_VERSION=7.57.0 \ + PS_VERSION=6.2.0 WORKDIR /curl ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab -RUN expand /R curl.cab /F:* . +ADD https://github.com/PowerShell/PowerShell/releases/download/v$PS_VERSION/PowerShell-$PS_VERSION-win-x64.zip /PowerShell/powershell.zip +ADD https://eternallybored.org/misc/netcat/netcat-win32-1.12.zip /netcat//netcat.zip + +USER ContainerAdministrator +RUN expand /R curl.cab /F:* . &\ + cd C:\PowerShell &\ + tar.exe -xf powershell.zip &\ + del powershell.zip &\ + mklink powershell.exe pwsh.exe &\ + cd C:\netcat &\ + tar.exe -xf netcat.zip &\ + del netcat.zip FROM $BASEIMAGE -COPY --from=prep /curl/AMD64 /curl -COPY --from=prep /curl/CURL.LIC /curl +COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/ +COPY --from=prep ["/PowerShell", "/Program Files/PowerShell"] +COPY --from=prep /netcat/nc64.exe /bin/nc.exe ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe -ADD https://github.com/diegocr/netcat/raw/master/nc.exe /bin/nc.exe ADD hostname /bin/hostname.exe + USER ContainerAdministrator RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe -RUN setx /M PATH "C:\bin;C:\curl\;%PATH%" &\ +# Set the path +RUN setx /M PATH "C:\bin;C:\curl\;%PATH%;C:\Program Files\PowerShell;" &\ mkdir C:\tmp + +# Copy PowerShell Core from the installer container +ENV ProgramFiles="C:\Program Files" \ + # set a fixed location for the Module analysis cache + LOCALAPPDATA="C:\Users\ContainerAdministrator\AppData\Local" \ + PSModuleAnalysisCachePath="$LOCALAPPDATA\Microsoft\Windows\PowerShell\docker\ModuleAnalysisCache" \ + # Persist %PSCORE% ENV variable for user convenience + PSCORE="$ProgramFiles\PowerShell\pwsh.exe" + +# intialize powershell module cache +RUN powershell \ + -NoLogo \ + -NoProfile \ + -Command " \ + $stopTime = (get-date).AddMinutes(15); \ + $ErrorActionPreference = 'Stop' ; \ + $ProgressPreference = 'SilentlyContinue' ; \ + while(!(Test-Path -Path $env:PSModuleAnalysisCachePath)) { \ + Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \ + if((get-date) -gt $stopTime) { throw 'timout expired'} \ + Start-Sleep -Seconds 6 ; \ + }" + ENTRYPOINT ["cmd.exe", "/s", "/c"] diff --git a/test/images/image-util.sh b/test/images/image-util.sh index 6fb6ae294cd..beeb72085e9 100755 --- a/test/images/image-util.sh +++ b/test/images/image-util.sh @@ -158,7 +158,7 @@ build() { docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \ --tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \ -H "${REMOTE_DOCKER_URL}" build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" \ - --build-arg BASEIMAGE="${BASEIMAGE}" -f $dockerfile_name . + --build-arg BASEIMAGE="${BASEIMAGE}" --build-arg REGISTRY="${REGISTRY}" -f $dockerfile_name . fi popd done @@ -256,7 +256,7 @@ if [[ "${WHAT}" == "all-conformance" ]]; then # no point in rebuilding all of them every time. This will only build the Conformance-related images. # Discussed during Conformance Office Hours Meeting (2019.12.17): # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh - conformance_images=("busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver") + conformance_images=("windows-image-builder-helper" "busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver") for image in "${conformance_images[@]}"; do eval "${TASK}" "${image}" done diff --git a/test/images/windows-image-builder-helper/BASEIMAGE b/test/images/windows-image-builder-helper/BASEIMAGE new file mode 100644 index 00000000000..ce71d103b81 --- /dev/null +++ b/test/images/windows-image-builder-helper/BASEIMAGE @@ -0,0 +1,3 @@ +windows/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019 +windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903 +windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909 diff --git a/test/images/windows-image-builder-helper/Dockerfile_windows b/test/images/windows-image-builder-helper/Dockerfile_windows new file mode 100644 index 00000000000..c9cc2c1c0ce --- /dev/null +++ b/test/images/windows-image-builder-helper/Dockerfile_windows @@ -0,0 +1,51 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASEIMAGE +from $BASEIMAGE as prep + +ENV CURL_VERSION=7.57.0 +WORKDIR /curl +ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab +ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe + +USER ContainerAdministrator +RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe +RUN expand /R curl.cab /F:* . &\ + del C:\curl\curl.cab &\ + setx /M PATH "C:\bin;C:\curl\;%PATH%" + +# download bind and prepare a folder just for dig. +RUN powershell -Command "\ + curl.exe 'https://downloads.isc.org/isc/bind9/9.14.10/BIND9.14.10.x64.zip' -o /bind.zip; \ + Expand-Archive -Path C:\bind.zip -DestinationPath C:\bind; \ + $s = [System.Diagnostics.Process]::Start('C:\bind\vcredist_x64.exe', '/quiet'); \ + $s.WaitForExit(); \ + mkdir C:\dig; \ + cp C:\bind\dig.exe C:\dig\; \ + cp C:\bind\nslookup.exe C:\dig\; \ + cp C:\bind\*.dll C:\dig\; \ + cp C:\Windows\System32\vcruntime140.dll C:\dig\; \ + rm C:\bind.zip;" + +FROM $BASEIMAGE + +COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/ +COPY --from=prep /bin /bin +COPY --from=prep /dig /dig + +ENV chocolateyUseWindowsCompression false +RUN powershell -Command "\ + iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \ + choco feature disable --name showDownloadProgress; diff --git a/test/images/windows-image-builder-helper/VERSION b/test/images/windows-image-builder-helper/VERSION new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/test/images/windows-image-builder-helper/VERSION @@ -0,0 +1 @@ +1.0