test images: uses nanoserver

Using Windows nanoserver container images as a base instead of the current
Windows servercore image will reduce the image size by about ~10x.

However, the nanoserver image lacks several things we need:
- netapi32.dll
- powershell
- certain powershell commands
- chocolatey cannot be used

When building the nanoserver images, we are going to use a Windows servercore helper,
in which we are going to install the necessary dependencies, and then copy them over
to our nanoserver image, including necessary DLLs.

Other notable changes include:
- switch from wget to curl (wget was a powershell alias).
- implement in code getting the DNS suffix list and DNS server list.
- reimplement getting file permissions for mounttest.

build/dependencies.yaml

@@ -1,7 +1,7 @@
 dependencies:
   # agnhost: bump this one first
   - name: "agnhost"
-    version: "2.22"
+    version: "2.23"
     refPaths:
     - path: test/images/agnhost/VERSION
       match: \d.\d

test/images/agnhost/Dockerfile_windows

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 ARG BASEIMAGE
+ARG REGISTRY
+FROM $REGISTRY/windows-image-builder-helper:1.0 as helper
 FROM $BASEIMAGE
 
 # from dnsutils image
@@ -24,18 +26,17 @@ FROM $BASEIMAGE
 # - curl, nc: used by a lot of e2e tests (inherited from BASEIMAGE)
 # from iperf image
 # install necessary packages: iperf
-ENV chocolateyUseWindowsCompression false
+COPY --from=helper /dig /dig
+COPY --from=helper /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll
+
+RUN setx /M PATH "C:\dig\;%PATH%
 RUN powershell -Command "\
-    iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
+    curl.exe -L 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -o C:\coredns.tgz;\
-    choco feature disable --name showDownloadProgress; \
+    tar.exe -xzvf C:\coredns.tgz;\
-    choco install bind-toolsonly --version 9.10.3 -y
-RUN powershell -Command "\
-    wget -uri 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -OutFile C:\coredns.tgz;\
-    tar -xzvf C:\coredns.tgz;\
     Remove-Item C:\coredns.tgz"
 
 RUN powershell -Command "\
-    wget -uri 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -OutFile C:\iperf.zip;\
+    curl.exe -L 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -o C:\iperf.zip;\
     Expand-Archive -Path C:\iperf.zip -DestinationPath C:\ -Force;\
     Rename-Item C:\iperf-2.0.9-win64 C:\iperf;\
     Remove-Item C:\iperf.zip"
@@ -53,6 +54,9 @@ RUN mkdir C:\uploads
 ADD porter/localhost.crt localhost.crt
 ADD porter/localhost.key localhost.key
 
+# from mounttest
+ADD mounttest/filePermissions.ps1 filePermissions.ps1
+
 ADD agnhost agnhost
 
 # needed for the entrypoint-tester related tests. Some of the entrypoint-tester related tests

test/images/agnhost/VERSION

@@ -1 +1 @@
-2.22
+2.23

test/images/agnhost/agnhost.go

@@ -51,7 +51,7 @@ import (
 func main() {
 	rootCmd := &cobra.Command{
 		Use:     "app",
-		Version: "2.22",
+		Version: "2.23",
 	}
 
 	rootCmd.AddCommand(auditproxy.CmdAuditProxy)

test/images/agnhost/dns/BUILD

@@ -15,7 +15,13 @@ go_library(
     importpath = "k8s.io/kubernetes/test/images/agnhost/dns",
     deps = [
         "//vendor/github.com/spf13/cobra:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//vendor/golang.org/x/sys/windows:go_default_library",
+            "//vendor/golang.org/x/sys/windows/registry:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(

test/images/agnhost/dns/dns_windows.go

@@ -1,3 +1,5 @@
+// +build windows
+
 /*
 Copyright 2019 The Kubernetes Authors.
 
@@ -17,40 +19,131 @@ limitations under the License.
 package dns
 
 import (
-	"bytes"
+	"fmt"
-	"os/exec"
 	"strings"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+	"golang.org/x/sys/windows/registry"
 )
 
-const etcHostsFile = "C:/Windows/System32/drivers/etc/hosts"
+const (
+	etcHostsFile      = "C:/Windows/System32/drivers/etc/hosts"
+	netRegistry       = `System\CurrentControlSet\Services\TCPIP\Parameters`
+	netIfacesRegistry = `System\CurrentControlSet\Services\TCPIP\Parameters\Interfaces`
+	maxHostnameLen    = 128
+	maxDomainNameLen  = 128
+	maxScopeIDLen     = 256
+)
 
+// FixedInfo information: https://docs.microsoft.com/en-us/windows/win32/api/iptypes/ns-iptypes-fixed_info_w2ksp1
+type FixedInfo struct {
+	HostName         [maxHostnameLen + 4]byte
+	DomainName       [maxDomainNameLen + 4]byte
+	CurrentDNSServer *syscall.IpAddrString
+	DNSServerList    syscall.IpAddrString
+	NodeType         uint32
+	ScopeID          [maxScopeIDLen + 4]byte
+	EnableRouting    uint32
+	EnableProxy      uint32
+	EnableDNS        uint32
+}
+
+var (
+	// GetNetworkParams can be found in iphlpapi.dll
+	// see: https://docs.microsoft.com/en-us/windows/win32/api/iphlpapi/nf-iphlpapi-getnetworkparams?redirectedfrom=MSDN
+	iphlpapidll          = windows.MustLoadDLL("iphlpapi.dll")
+	procGetNetworkParams = iphlpapidll.MustFindProc("GetNetworkParams")
+)
+
+func elemInList(elem string, list []string) bool {
+	for _, e := range list {
+		if e == elem {
+			return true
+		}
+	}
+	return false
+}
+
+func getRegistryValue(reg, key string) string {
+	regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, reg, registry.QUERY_VALUE)
+	if err != nil {
+		return ""
+	}
+	defer regKey.Close()
+
+	regValue, _, err := regKey.GetStringValue(key)
+	if err != nil {
+		return ""
+	}
+	return regValue
+}
+
+// getDNSSuffixList reads DNS config file and returns the list of configured DNS suffixes
 func getDNSSuffixList() []string {
-	output := runCommand("powershell", "-Command", "(Get-DnsClient)[0].SuffixSearchList")
+	// We start with the general suffix list that apply to all network connections.
-	if len(output) > 0 {
+	allSuffixes := []string{}
-		return strings.Split(output, "\r\n")
+	suffixes := getRegistryValue(netRegistry, "SearchList")
+	if suffixes != "" {
+		allSuffixes = strings.Split(suffixes, ",")
 	}
 
-	panic("Could not find DNS search list!")
+	// Then we append the network-specific DNS suffix lists.
-}
+	regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, netIfacesRegistry, registry.ENUMERATE_SUB_KEYS)
-
-func getDNSServerList() []string {
-	output := runCommand("powershell", "-Command", "(Get-DnsClientServerAddress).ServerAddresses")
-	if len(output) > 0 {
-		return strings.Split(output, "\r\n")
-	}
-
-	panic("Could not find DNS Server list!")
-}
-
-func runCommand(name string, arg ...string) string {
-	var out bytes.Buffer
-	cmd := exec.Command(name, arg...)
-	cmd.Stdout = &out
-
-	err := cmd.Run()
 	if err != nil {
 		panic(err)
 	}
+	defer regKey.Close()
 
-	return strings.TrimSpace(out.String())
+	ifaces, err := regKey.ReadSubKeyNames(0)
+	if err != nil {
+		panic(err)
+	}
+	for _, iface := range ifaces {
+		suffixes := getRegistryValue(fmt.Sprintf("%s\\%s", netIfacesRegistry, iface), "SearchList")
+		if suffixes == "" {
+			continue
+		}
+		for _, suffix := range strings.Split(suffixes, ",") {
+			if !elemInList(suffix, allSuffixes) {
+				allSuffixes = append(allSuffixes, suffix)
+			}
+		}
+	}
+
+	return allSuffixes
+}
+
+func getNetworkParams() *FixedInfo {
+	// We don't know how big we should make the byte buffer, but the call will tell us by
+	// setting the size afterwards.
+	var size int
+	buffer := make([]byte, 1)
+	procGetNetworkParams.Call(
+		uintptr(unsafe.Pointer(&buffer[0])),
+		uintptr(unsafe.Pointer(&size)),
+	)
+
+	buffer = make([]byte, size)
+	procGetNetworkParams.Call(
+		uintptr(unsafe.Pointer(&buffer[0])),
+		uintptr(unsafe.Pointer(&size)),
+	)
+
+	info := (*FixedInfo)(unsafe.Pointer(&buffer[0]))
+	return info
+}
+
+func getDNSServerList() []string {
+	dnsServerList := []string{}
+	fixedInfo := getNetworkParams()
+	list := &(fixedInfo.DNSServerList)
+
+	for list != nil {
+		dnsServer := strings.TrimRight(string(list.IpAddress.String[:]), "\x00")
+		dnsServerList = append(dnsServerList, dnsServer)
+		list = list.Next
+	}
+	return dnsServerList
 }

test/images/agnhost/mounttest/filePermissions.ps1

@@ -28,59 +28,41 @@ $EXECUTE_PERMISSIONS = 0x0001 -bor 0x0020
 
 
 function GetFilePermissions($path) {
-    $objPath = "Win32_LogicalFileSecuritySetting='$path'"
+    $fileAcl = Get-Acl -Path $path
-    $output = Invoke-WmiMethod -Namespace root/cimv2 -Path $objPath -Name GetSecurityDescriptor
+    $fileOwner = $fileAcl.Owner
-
+    $fileGroup = $fileAcl.Group
-    if ($output.ReturnValue -ne 0) {
-        $retVal = $output.ReturnValue
-        Write-Error "GetSecurityDescriptor invocation failed with code: $retVal"
-        exit 1
-    }
-
-    $fileSD = $output.Descriptor
-    $fileOwnerGroup = $fileSD.Group
-    $fileOwner = $fileSD.Owner
-
-    if ($fileOwnerGroup.Name -eq $null -and $fileOwnerGroup.Domain -eq $null) {
-        # the file owner's group is not recognized. Check if the Owner itself is
-        # a group, and if so, default the group to it.
-        net user $fileOwner.Name > $null 2> $null
-        if (-not $?) {
-            $fileOwnerGroup = $fileOwner
-        }
-
-    }
 
     $userMask = 0
     $groupMask = 0
     $otherMask = 0
 
-    foreach ($ace in $fileSD.DACL) {
+    foreach ($rule in $fileAcl.Access) {
-        $mask = 0
+        if ($rule.AccessControlType -ne [Security.AccessControl.AccessControlType]::Allow) {
-        if ($ace.AceType -ne 0) {
+            # not an allow rule, skipping.
-            # not an Allow ACE, skip.
             continue
         }
 
+        $mask = 0
+        $rights = $rule.FileSystemRights.value__
         # convert mask.
-        if ( ($ace.AccessMask -band $READ_PERMISSIONS) -eq $READ_PERMISSIONS ) {
+        if ( ($rights -band $READ_PERMISSIONS) -eq $READ_PERMISSIONS ) {
             $mask = $mask -bor 4
         }
-        if ( ($ace.AccessMask -band $WRITE_PERMISSIONS) -eq $WRITE_PERMISSIONS ) {
+        if ( ($rights -band $WRITE_PERMISSIONS) -eq $WRITE_PERMISSIONS ) {
             $mask = $mask -bor 2
         }
-        if ( ($ace.AccessMask -band $EXECUTE_PERMISSIONS) -eq $EXECUTE_PERMISSIONS ) {
+        if ( ($rights -band $EXECUTE_PERMISSIONS) -eq $EXECUTE_PERMISSIONS ) {
             $mask = $mask -bor 1
         }
 
         # detect mask type.
-        if ($ace.Trustee.Equals($fileOwner)) {
+        if ($rule.IdentityReference.Value.Equals($fileOwner)) {
             $userMask = $mask
         }
-        if ($ace.Trustee.Equals($fileOwnerGroup)) {
+        if ($rule.IdentityReference.Value.Equals($fileGroup)) {
             $groupMask = $mask
         }
-        if ($ace.Trustee.Name.ToLower() -eq "users") {
+        if ($rule.IdentityReference.Value.ToLower().Contains("users")) {
             $otherMask = $mask
         }
     }

test/images/busybox/BASEIMAGE

@@ -1,3 +1,3 @@
-windows/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019
+windows/amd64/1809=mcr.microsoft.com/windows/nanoserver:1809
-windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903
+windows/amd64/1903=mcr.microsoft.com/windows/nanoserver:1903
-windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909
+windows/amd64/1909=mcr.microsoft.com/windows/nanoserver:1909

test/images/busybox/Dockerfile_windows

@@ -15,20 +15,57 @@
 ARG BASEIMAGE
 from $BASEIMAGE as prep
 
-ENV CURL_VERSION 7.57.0
+ENV CURL_VERSION=7.57.0 \
+    PS_VERSION=6.2.0
 WORKDIR /curl
 ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab
-RUN expand /R curl.cab /F:* .
+ADD https://github.com/PowerShell/PowerShell/releases/download/v$PS_VERSION/PowerShell-$PS_VERSION-win-x64.zip /PowerShell/powershell.zip
+ADD https://eternallybored.org/misc/netcat/netcat-win32-1.12.zip /netcat//netcat.zip
+
+USER ContainerAdministrator
+RUN expand /R curl.cab /F:* . &\
+    cd C:\PowerShell &\
+    tar.exe -xf powershell.zip &\
+    del powershell.zip &\
+    mklink powershell.exe pwsh.exe &\
+    cd C:\netcat &\
+    tar.exe -xf netcat.zip &\
+    del netcat.zip
 
 FROM $BASEIMAGE
 
-COPY --from=prep /curl/AMD64 /curl
+COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/
-COPY --from=prep /curl/CURL.LIC /curl
+COPY --from=prep ["/PowerShell", "/Program Files/PowerShell"]
+COPY --from=prep /netcat/nc64.exe /bin/nc.exe
 ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe
-ADD https://github.com/diegocr/netcat/raw/master/nc.exe /bin/nc.exe
 ADD hostname /bin/hostname.exe
+
 USER ContainerAdministrator
 RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe
-RUN setx /M PATH "C:\bin;C:\curl\;%PATH%" &\
+# Set the path
+RUN setx /M PATH "C:\bin;C:\curl\;%PATH%;C:\Program Files\PowerShell;" &\
     mkdir C:\tmp
+
+# Copy PowerShell Core from the installer container
+ENV ProgramFiles="C:\Program Files" \
+    # set a fixed location for the Module analysis cache
+    LOCALAPPDATA="C:\Users\ContainerAdministrator\AppData\Local" \
+    PSModuleAnalysisCachePath="$LOCALAPPDATA\Microsoft\Windows\PowerShell\docker\ModuleAnalysisCache" \
+    # Persist %PSCORE% ENV variable for user convenience
+    PSCORE="$ProgramFiles\PowerShell\pwsh.exe"
+
+# intialize powershell module cache
+RUN powershell \
+        -NoLogo \
+        -NoProfile \
+        -Command " \
+          $stopTime = (get-date).AddMinutes(15); \
+          $ErrorActionPreference = 'Stop' ; \
+          $ProgressPreference = 'SilentlyContinue' ; \
+          while(!(Test-Path -Path $env:PSModuleAnalysisCachePath)) {  \
+            Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \
+            if((get-date) -gt $stopTime) { throw 'timout expired'} \
+            Start-Sleep -Seconds 6 ; \
+          }"
+
 ENTRYPOINT ["cmd.exe", "/s", "/c"]

test/images/image-util.sh

@@ -158,7 +158,7 @@ build() {
       docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \
         --tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \
         -H "${REMOTE_DOCKER_URL}" build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" \
-        --build-arg BASEIMAGE="${BASEIMAGE}" -f $dockerfile_name .
+        --build-arg BASEIMAGE="${BASEIMAGE}" --build-arg REGISTRY="${REGISTRY}" -f $dockerfile_name .
     fi
     popd
   done
@@ -256,7 +256,7 @@ if [[ "${WHAT}" == "all-conformance" ]]; then
   # no point in rebuilding all of them every time. This will only build the Conformance-related images.
   # Discussed during Conformance Office Hours Meeting (2019.12.17):
   # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh
-  conformance_images=("busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
+  conformance_images=("windows-image-builder-helper" "busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
   for image in "${conformance_images[@]}"; do
     eval "${TASK}" "${image}"
   done

test/images/windows-image-builder-helper/BASEIMAGE

@@ -0,0 +1,3 @@
+windows/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019
+windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903
+windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909

test/images/windows-image-builder-helper/Dockerfile_windows

@@ -0,0 +1,51 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASEIMAGE
+from $BASEIMAGE as prep
+
+ENV CURL_VERSION=7.57.0
+WORKDIR /curl
+ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab
+ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe
+
+USER ContainerAdministrator
+RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe
+RUN expand /R curl.cab /F:* . &\
+    del C:\curl\curl.cab &\
+    setx /M PATH "C:\bin;C:\curl\;%PATH%"
+
+# download bind and prepare a folder just for dig.
+RUN powershell -Command "\
+      curl.exe 'https://downloads.isc.org/isc/bind9/9.14.10/BIND9.14.10.x64.zip' -o /bind.zip; \
+      Expand-Archive -Path C:\bind.zip -DestinationPath C:\bind; \
+      $s = [System.Diagnostics.Process]::Start('C:\bind\vcredist_x64.exe', '/quiet'); \
+      $s.WaitForExit(); \
+      mkdir C:\dig; \
+      cp C:\bind\dig.exe C:\dig\; \
+      cp C:\bind\nslookup.exe C:\dig\; \
+      cp C:\bind\*.dll C:\dig\; \
+      cp C:\Windows\System32\vcruntime140.dll C:\dig\; \
+      rm C:\bind.zip;"
+
+FROM $BASEIMAGE
+
+COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/
+COPY --from=prep /bin /bin
+COPY --from=prep /dig /dig
+
+ENV chocolateyUseWindowsCompression false
+RUN powershell -Command "\
+    iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
+    choco feature disable --name showDownloadProgress;

test/images/windows-image-builder-helper/VERSION

@@ -0,0 +1 @@
+1.0