test images: uses nanoserver

Using Windows nanoserver container images as a base instead of the current
Windows servercore image will reduce the image size by about ~10x.

However, the nanoserver image lacks several things we need:
- netapi32.dll
- powershell
- certain powershell commands
- chocolatey cannot be used

When building the nanoserver images, we are going to use a Windows servercore helper,
in which we are going to install the necessary dependencies, and then copy them over
to our nanoserver image, including necessary DLLs.

Other notable changes include:
- switch from wget to curl (wget was a powershell alias).
- implement in code getting the DNS suffix list and DNS server list.
- reimplement getting file permissions for mounttest.
This commit is contained in:
Claudiu Belu 2020-02-13 15:23:50 +00:00
parent 8d30a5f136
commit 46c820e793
13 changed files with 257 additions and 80 deletions

View File

@ -1,7 +1,7 @@
dependencies:
# agnhost: bump this one first
- name: "agnhost"
version: "2.22"
version: "2.23"
refPaths:
- path: test/images/agnhost/VERSION
match: \d.\d

View File

@ -13,6 +13,8 @@
# limitations under the License.
ARG BASEIMAGE
ARG REGISTRY
FROM $REGISTRY/windows-image-builder-helper:1.0 as helper
FROM $BASEIMAGE
# from dnsutils image
@ -24,18 +26,17 @@ FROM $BASEIMAGE
# - curl, nc: used by a lot of e2e tests (inherited from BASEIMAGE)
# from iperf image
# install necessary packages: iperf
ENV chocolateyUseWindowsCompression false
COPY --from=helper /dig /dig
COPY --from=helper /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll
RUN setx /M PATH "C:\dig\;%PATH%
RUN powershell -Command "\
iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
choco feature disable --name showDownloadProgress; \
choco install bind-toolsonly --version 9.10.3 -y
RUN powershell -Command "\
wget -uri 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -OutFile C:\coredns.tgz;\
tar -xzvf C:\coredns.tgz;\
curl.exe -L 'https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz' -o C:\coredns.tgz;\
tar.exe -xzvf C:\coredns.tgz;\
Remove-Item C:\coredns.tgz"
RUN powershell -Command "\
wget -uri 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -OutFile C:\iperf.zip;\
curl.exe -L 'https://iperf.fr/download/windows/iperf-2.0.9-win64.zip' -o C:\iperf.zip;\
Expand-Archive -Path C:\iperf.zip -DestinationPath C:\ -Force;\
Rename-Item C:\iperf-2.0.9-win64 C:\iperf;\
Remove-Item C:\iperf.zip"
@ -53,6 +54,9 @@ RUN mkdir C:\uploads
ADD porter/localhost.crt localhost.crt
ADD porter/localhost.key localhost.key
# from mounttest
ADD mounttest/filePermissions.ps1 filePermissions.ps1
ADD agnhost agnhost
# needed for the entrypoint-tester related tests. Some of the entrypoint-tester related tests

View File

@ -1 +1 @@
2.22
2.23

View File

@ -51,7 +51,7 @@ import (
func main() {
rootCmd := &cobra.Command{
Use: "app",
Version: "2.22",
Version: "2.23",
}
rootCmd.AddCommand(auditproxy.CmdAuditProxy)

View File

@ -15,7 +15,13 @@ go_library(
importpath = "k8s.io/kubernetes/test/images/agnhost/dns",
deps = [
"//vendor/github.com/spf13/cobra:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/golang.org/x/sys/windows:go_default_library",
"//vendor/golang.org/x/sys/windows/registry:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(

View File

@ -1,3 +1,5 @@
// +build windows
/*
Copyright 2019 The Kubernetes Authors.
@ -17,40 +19,131 @@ limitations under the License.
package dns
import (
"bytes"
"os/exec"
"fmt"
"strings"
"syscall"
"unsafe"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/registry"
)
const etcHostsFile = "C:/Windows/System32/drivers/etc/hosts"
const (
etcHostsFile = "C:/Windows/System32/drivers/etc/hosts"
netRegistry = `System\CurrentControlSet\Services\TCPIP\Parameters`
netIfacesRegistry = `System\CurrentControlSet\Services\TCPIP\Parameters\Interfaces`
maxHostnameLen = 128
maxDomainNameLen = 128
maxScopeIDLen = 256
)
// FixedInfo information: https://docs.microsoft.com/en-us/windows/win32/api/iptypes/ns-iptypes-fixed_info_w2ksp1
type FixedInfo struct {
HostName [maxHostnameLen + 4]byte
DomainName [maxDomainNameLen + 4]byte
CurrentDNSServer *syscall.IpAddrString
DNSServerList syscall.IpAddrString
NodeType uint32
ScopeID [maxScopeIDLen + 4]byte
EnableRouting uint32
EnableProxy uint32
EnableDNS uint32
}
var (
// GetNetworkParams can be found in iphlpapi.dll
// see: https://docs.microsoft.com/en-us/windows/win32/api/iphlpapi/nf-iphlpapi-getnetworkparams?redirectedfrom=MSDN
iphlpapidll = windows.MustLoadDLL("iphlpapi.dll")
procGetNetworkParams = iphlpapidll.MustFindProc("GetNetworkParams")
)
func elemInList(elem string, list []string) bool {
for _, e := range list {
if e == elem {
return true
}
}
return false
}
func getRegistryValue(reg, key string) string {
regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, reg, registry.QUERY_VALUE)
if err != nil {
return ""
}
defer regKey.Close()
regValue, _, err := regKey.GetStringValue(key)
if err != nil {
return ""
}
return regValue
}
// getDNSSuffixList reads DNS config file and returns the list of configured DNS suffixes
func getDNSSuffixList() []string {
output := runCommand("powershell", "-Command", "(Get-DnsClient)[0].SuffixSearchList")
if len(output) > 0 {
return strings.Split(output, "\r\n")
// We start with the general suffix list that apply to all network connections.
allSuffixes := []string{}
suffixes := getRegistryValue(netRegistry, "SearchList")
if suffixes != "" {
allSuffixes = strings.Split(suffixes, ",")
}
panic("Could not find DNS search list!")
}
func getDNSServerList() []string {
output := runCommand("powershell", "-Command", "(Get-DnsClientServerAddress).ServerAddresses")
if len(output) > 0 {
return strings.Split(output, "\r\n")
}
panic("Could not find DNS Server list!")
}
func runCommand(name string, arg ...string) string {
var out bytes.Buffer
cmd := exec.Command(name, arg...)
cmd.Stdout = &out
err := cmd.Run()
// Then we append the network-specific DNS suffix lists.
regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, netIfacesRegistry, registry.ENUMERATE_SUB_KEYS)
if err != nil {
panic(err)
}
defer regKey.Close()
return strings.TrimSpace(out.String())
ifaces, err := regKey.ReadSubKeyNames(0)
if err != nil {
panic(err)
}
for _, iface := range ifaces {
suffixes := getRegistryValue(fmt.Sprintf("%s\\%s", netIfacesRegistry, iface), "SearchList")
if suffixes == "" {
continue
}
for _, suffix := range strings.Split(suffixes, ",") {
if !elemInList(suffix, allSuffixes) {
allSuffixes = append(allSuffixes, suffix)
}
}
}
return allSuffixes
}
func getNetworkParams() *FixedInfo {
// We don't know how big we should make the byte buffer, but the call will tell us by
// setting the size afterwards.
var size int
buffer := make([]byte, 1)
procGetNetworkParams.Call(
uintptr(unsafe.Pointer(&buffer[0])),
uintptr(unsafe.Pointer(&size)),
)
buffer = make([]byte, size)
procGetNetworkParams.Call(
uintptr(unsafe.Pointer(&buffer[0])),
uintptr(unsafe.Pointer(&size)),
)
info := (*FixedInfo)(unsafe.Pointer(&buffer[0]))
return info
}
func getDNSServerList() []string {
dnsServerList := []string{}
fixedInfo := getNetworkParams()
list := &(fixedInfo.DNSServerList)
for list != nil {
dnsServer := strings.TrimRight(string(list.IpAddress.String[:]), "\x00")
dnsServerList = append(dnsServerList, dnsServer)
list = list.Next
}
return dnsServerList
}

View File

@ -28,59 +28,41 @@ $EXECUTE_PERMISSIONS = 0x0001 -bor 0x0020
function GetFilePermissions($path) {
$objPath = "Win32_LogicalFileSecuritySetting='$path'"
$output = Invoke-WmiMethod -Namespace root/cimv2 -Path $objPath -Name GetSecurityDescriptor
if ($output.ReturnValue -ne 0) {
$retVal = $output.ReturnValue
Write-Error "GetSecurityDescriptor invocation failed with code: $retVal"
exit 1
}
$fileSD = $output.Descriptor
$fileOwnerGroup = $fileSD.Group
$fileOwner = $fileSD.Owner
if ($fileOwnerGroup.Name -eq $null -and $fileOwnerGroup.Domain -eq $null) {
# the file owner's group is not recognized. Check if the Owner itself is
# a group, and if so, default the group to it.
net user $fileOwner.Name > $null 2> $null
if (-not $?) {
$fileOwnerGroup = $fileOwner
}
}
$fileAcl = Get-Acl -Path $path
$fileOwner = $fileAcl.Owner
$fileGroup = $fileAcl.Group
$userMask = 0
$groupMask = 0
$otherMask = 0
foreach ($ace in $fileSD.DACL) {
$mask = 0
if ($ace.AceType -ne 0) {
# not an Allow ACE, skip.
foreach ($rule in $fileAcl.Access) {
if ($rule.AccessControlType -ne [Security.AccessControl.AccessControlType]::Allow) {
# not an allow rule, skipping.
continue
}
$mask = 0
$rights = $rule.FileSystemRights.value__
# convert mask.
if ( ($ace.AccessMask -band $READ_PERMISSIONS) -eq $READ_PERMISSIONS ) {
if ( ($rights -band $READ_PERMISSIONS) -eq $READ_PERMISSIONS ) {
$mask = $mask -bor 4
}
if ( ($ace.AccessMask -band $WRITE_PERMISSIONS) -eq $WRITE_PERMISSIONS ) {
if ( ($rights -band $WRITE_PERMISSIONS) -eq $WRITE_PERMISSIONS ) {
$mask = $mask -bor 2
}
if ( ($ace.AccessMask -band $EXECUTE_PERMISSIONS) -eq $EXECUTE_PERMISSIONS ) {
if ( ($rights -band $EXECUTE_PERMISSIONS) -eq $EXECUTE_PERMISSIONS ) {
$mask = $mask -bor 1
}
# detect mask type.
if ($ace.Trustee.Equals($fileOwner)) {
if ($rule.IdentityReference.Value.Equals($fileOwner)) {
$userMask = $mask
}
if ($ace.Trustee.Equals($fileOwnerGroup)) {
if ($rule.IdentityReference.Value.Equals($fileGroup)) {
$groupMask = $mask
}
if ($ace.Trustee.Name.ToLower() -eq "users") {
if ($rule.IdentityReference.Value.ToLower().Contains("users")) {
$otherMask = $mask
}
}

View File

@ -1,3 +1,3 @@
windows/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019
windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903
windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909
windows/amd64/1809=mcr.microsoft.com/windows/nanoserver:1809
windows/amd64/1903=mcr.microsoft.com/windows/nanoserver:1903
windows/amd64/1909=mcr.microsoft.com/windows/nanoserver:1909

View File

@ -15,20 +15,57 @@
ARG BASEIMAGE
from $BASEIMAGE as prep
ENV CURL_VERSION 7.57.0
ENV CURL_VERSION=7.57.0 \
PS_VERSION=6.2.0
WORKDIR /curl
ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab
RUN expand /R curl.cab /F:* .
ADD https://github.com/PowerShell/PowerShell/releases/download/v$PS_VERSION/PowerShell-$PS_VERSION-win-x64.zip /PowerShell/powershell.zip
ADD https://eternallybored.org/misc/netcat/netcat-win32-1.12.zip /netcat//netcat.zip
USER ContainerAdministrator
RUN expand /R curl.cab /F:* . &\
cd C:\PowerShell &\
tar.exe -xf powershell.zip &\
del powershell.zip &\
mklink powershell.exe pwsh.exe &\
cd C:\netcat &\
tar.exe -xf netcat.zip &\
del netcat.zip
FROM $BASEIMAGE
COPY --from=prep /curl/AMD64 /curl
COPY --from=prep /curl/CURL.LIC /curl
COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/
COPY --from=prep ["/PowerShell", "/Program Files/PowerShell"]
COPY --from=prep /netcat/nc64.exe /bin/nc.exe
ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe
ADD https://github.com/diegocr/netcat/raw/master/nc.exe /bin/nc.exe
ADD hostname /bin/hostname.exe
USER ContainerAdministrator
RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe
RUN setx /M PATH "C:\bin;C:\curl\;%PATH%" &\
# Set the path
RUN setx /M PATH "C:\bin;C:\curl\;%PATH%;C:\Program Files\PowerShell;" &\
mkdir C:\tmp
# Copy PowerShell Core from the installer container
ENV ProgramFiles="C:\Program Files" \
# set a fixed location for the Module analysis cache
LOCALAPPDATA="C:\Users\ContainerAdministrator\AppData\Local" \
PSModuleAnalysisCachePath="$LOCALAPPDATA\Microsoft\Windows\PowerShell\docker\ModuleAnalysisCache" \
# Persist %PSCORE% ENV variable for user convenience
PSCORE="$ProgramFiles\PowerShell\pwsh.exe"
# intialize powershell module cache
RUN powershell \
-NoLogo \
-NoProfile \
-Command " \
$stopTime = (get-date).AddMinutes(15); \
$ErrorActionPreference = 'Stop' ; \
$ProgressPreference = 'SilentlyContinue' ; \
while(!(Test-Path -Path $env:PSModuleAnalysisCachePath)) { \
Write-Host "'Waiting for $env:PSModuleAnalysisCachePath'" ; \
if((get-date) -gt $stopTime) { throw 'timout expired'} \
Start-Sleep -Seconds 6 ; \
}"
ENTRYPOINT ["cmd.exe", "/s", "/c"]

View File

@ -158,7 +158,7 @@ build() {
docker --tlsverify --tlscacert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/ca.pem" \
--tlscert "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/cert.pem" --tlskey "${DOCKER_CERT_BASE_PATH}/.docker-${os_version}/key.pem" \
-H "${REMOTE_DOCKER_URL}" build --pull -t "${REGISTRY}/${image}:${TAG}-${os_name}-${arch}-${os_version}" \
--build-arg BASEIMAGE="${BASEIMAGE}" -f $dockerfile_name .
--build-arg BASEIMAGE="${BASEIMAGE}" --build-arg REGISTRY="${REGISTRY}" -f $dockerfile_name .
fi
popd
done
@ -256,7 +256,7 @@ if [[ "${WHAT}" == "all-conformance" ]]; then
# no point in rebuilding all of them every time. This will only build the Conformance-related images.
# Discussed during Conformance Office Hours Meeting (2019.12.17):
# https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh
conformance_images=("busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
conformance_images=("windows-image-builder-helper" "busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
for image in "${conformance_images[@]}"; do
eval "${TASK}" "${image}"
done

View File

@ -0,0 +1,3 @@
windows/amd64/1809=mcr.microsoft.com/windows/servercore:ltsc2019
windows/amd64/1903=mcr.microsoft.com/windows/servercore:1903
windows/amd64/1909=mcr.microsoft.com/windows/servercore:1909

View File

@ -0,0 +1,51 @@
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ARG BASEIMAGE
from $BASEIMAGE as prep
ENV CURL_VERSION=7.57.0
WORKDIR /curl
ADD https://skanthak.homepage.t-online.de/download/curl-$CURL_VERSION.cab curl.cab
ADD https://github.com/kubernetes-sigs/windows-testing/raw/master/images/busybox/busybox.exe /bin/busybox.exe
USER ContainerAdministrator
RUN FOR /f "tokens=*" %i IN ('C:\bin\busybox --list') DO mklink C:\bin\%i.exe C:\bin\busybox.exe
RUN expand /R curl.cab /F:* . &\
del C:\curl\curl.cab &\
setx /M PATH "C:\bin;C:\curl\;%PATH%"
# download bind and prepare a folder just for dig.
RUN powershell -Command "\
curl.exe 'https://downloads.isc.org/isc/bind9/9.14.10/BIND9.14.10.x64.zip' -o /bind.zip; \
Expand-Archive -Path C:\bind.zip -DestinationPath C:\bind; \
$s = [System.Diagnostics.Process]::Start('C:\bind\vcredist_x64.exe', '/quiet'); \
$s.WaitForExit(); \
mkdir C:\dig; \
cp C:\bind\dig.exe C:\dig\; \
cp C:\bind\nslookup.exe C:\dig\; \
cp C:\bind\*.dll C:\dig\; \
cp C:\Windows\System32\vcruntime140.dll C:\dig\; \
rm C:\bind.zip;"
FROM $BASEIMAGE
COPY --from=prep /curl/AMD64 /curl/CURL.LIC /curl/
COPY --from=prep /bin /bin
COPY --from=prep /dig /dig
ENV chocolateyUseWindowsCompression false
RUN powershell -Command "\
iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
choco feature disable --name showDownloadProgress;

View File

@ -0,0 +1 @@
1.0