diff --git a/build/dependencies.yaml b/build/dependencies.yaml index 8bef1ccce18..eb4fd33f46b 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -83,3 +83,5 @@ dependencies: match: CRI_TOOLS_VERSION = - path: cluster/gce/gci/configure.sh match: DEFAULT_CRICTL_VERSION= + - path: cluster/gce/windows/k8s-node-setup.psm1 + match: CRICTL_VERSION = diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index ddc097241b8..17cae504fc4 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -98,9 +98,7 @@ CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-} LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-} if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-containerd} - CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock} LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-ctr -n=k8s.io images import} - KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS} --runtime-cgroups=/system.slice/containerd.service" fi # MASTER_EXTRA_METADATA is the extra instance metadata on master instance separated by commas. diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 4d766a94c53..319185a8cd8 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -104,9 +104,7 @@ LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-} GCI_DOCKER_VERSION=${KUBE_GCI_DOCKER_VERSION:-} if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-containerd} - CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock} LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-ctr -n=k8s.io images import} - KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --runtime-cgroups=/system.slice/containerd.service" fi # MASTER_EXTRA_METADATA is the extra instance metadata on master instance separated by commas. diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index f777ed5bd74..e0e645bda7a 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -796,7 +796,12 @@ function construct-linux-kubelet-flags { fi if [[ "${CONTAINER_RUNTIME:-}" != "docker" ]]; then flags+=" --container-runtime=remote" + if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then + CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock} + flags+=" --runtime-cgroups=/system.slice/containerd.service" + fi fi + if [[ -n "${CONTAINER_RUNTIME_ENDPOINT:-}" ]]; then flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}" fi @@ -886,6 +891,14 @@ function construct-windows-kubelet-flags { # Force disable KubeletPodResources feature on Windows until #78628 is fixed. flags+=" --feature-gates=KubeletPodResources=false" + if [[ "${CONTAINER_RUNTIME:-}" != "docker" ]]; then + flags+=" --container-runtime=remote" + if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then + CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-npipe:////./pipe/containerd-containerd} + flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}" + fi + fi + KUBELET_ARGS="${flags}" } diff --git a/cluster/gce/windows/common.psm1 b/cluster/gce/windows/common.psm1 index a6d6ceee98f..663bf12685a 100644 --- a/cluster/gce/windows/common.psm1 +++ b/cluster/gce/windows/common.psm1 @@ -106,30 +106,33 @@ function Get-InstanceMetadataAttribute { return Get-InstanceMetadata "attributes/$Key" $Default } -function Validate-SHA1 { +function Validate-SHA { param( [parameter(Mandatory=$true)] [string]$Hash, - [parameter(Mandatory=$true)] [string]$Path + [parameter(Mandatory=$true)] [string]$Path, + [parameter(Mandatory=$true)] [string]$Algorithm ) - $actual = Get-FileHash -Path $Path -Algorithm SHA1 + $actual = Get-FileHash -Path $Path -Algorithm $Algorithm # Note: Powershell string comparisons are case-insensitive by default, and this # is important here because Linux shell scripts produce lowercase hashes but # Powershell Get-FileHash produces uppercase hashes. This must be case-insensitive # to work. if ($actual.Hash -ne $Hash) { - Log-Output "$Path corrupted, sha1 $actual doesn't match expected $Hash" - Throw ("$Path corrupted, sha1 $actual doesn't match expected $Hash") + Log-Output "$Path corrupted, $Algorithm $actual doesn't match expected $Hash" + Throw ("$Path corrupted, $Algorithm $actual doesn't match expected $Hash") } } # Attempts to download the file from URLs, trying each URL until it succeeds. # It will loop through the URLs list forever until it has a success. If # successful, it will write the file to OutFile. You can optionally provide a -# SHA1 Hash argument, in which case it will attempt to validate the downloaded -# file against the hash. +# Hash argument with an optional Algorithm, in which case it will attempt to +# validate the downloaded file against the hash. SHA1 will be used if Algorithm +# is not provided. function MustDownload-File { param ( [parameter(Mandatory=$false)] [string]$Hash, + [parameter(Mandatory=$false)] [string]$Algorithm = 'SHA1', [parameter(Mandatory=$true)] [string]$OutFile, [parameter(Mandatory=$true)] [System.Collections.Generic.List[String]]$URLs, [parameter(Mandatory=$false)] [System.Collections.IDictionary]$Headers = @{} @@ -156,13 +159,13 @@ function MustDownload-File { # Attempt to validate the hash if ($Hash) { Try { - Validate-SHA1 -Hash $Hash -Path $OutFile + Validate-SHA -Hash $Hash -Path $OutFile -Algorithm $Algorithm } Catch { $message = $_.Exception.ToString() Log-Output "Hash validation of $url failed. Will retry. Error: $message" continue } - Log-Output "Downloaded $url (SHA1 = $Hash)" + Log-Output "Downloaded $url ($Algorithm = $Hash)" return } Log-Output "Downloaded $url" diff --git a/cluster/gce/windows/configure.ps1 b/cluster/gce/windows/configure.ps1 index 434dace1e87..2fe2c32b6d0 100644 --- a/cluster/gce/windows/configure.ps1 +++ b/cluster/gce/windows/configure.ps1 @@ -131,8 +131,8 @@ try { Create-Directories Download-HelperScripts - Create-DockerRegistryKey - Configure-Dockerd + DownloadAndInstall-Crictl + Setup-ContainerRuntime DownloadAndInstall-KubernetesBinaries Create-NodePki Create-KubeletKubeconfig @@ -156,6 +156,7 @@ try { Verify-WorkerServices $config = New-FileRotationConfig + # TODO(random-liu): Generate containerd log into the log directory. Schedule-LogRotation -Pattern '.*\.log$' -Path ${env:LOGS_DIR} -RepetitionInterval $(New-Timespan -Hour 1) -Config $config Pull-InfraContainer diff --git a/cluster/gce/windows/k8s-node-setup.psm1 b/cluster/gce/windows/k8s-node-setup.psm1 index f9aa08f18b1..c0ed6f5aec7 100644 --- a/cluster/gce/windows/k8s-node-setup.psm1 +++ b/cluster/gce/windows/k8s-node-setup.psm1 @@ -209,7 +209,7 @@ function Fetch-KubeEnv { function Set_MachineEnvironmentVar { param ( [parameter(Mandatory=$true)] [string]$Key, - [parameter(Mandatory=$true)] [string]$Value + [parameter(Mandatory=$true)] [AllowEmptyString()] [string]$Value ) [Environment]::SetEnvironmentVariable($Key, $Value, "Machine") } @@ -218,7 +218,7 @@ function Set_MachineEnvironmentVar { function Set_CurrentShellEnvironmentVar { param ( [parameter(Mandatory=$true)] [string]$Key, - [parameter(Mandatory=$true)] [string]$Value + [parameter(Mandatory=$true)] [AllowEmptyString()] [string]$Value ) $expression = '$env:' + $Key + ' = "' + $Value + '"' Invoke-Expression ${expression} @@ -247,6 +247,9 @@ function Set-EnvironmentVars { "KUBELET_CERT_PATH" = ${kube_env}['PKI_DIR'] + '\kubelet.crt' "KUBELET_KEY_PATH" = ${kube_env}['PKI_DIR'] + '\kubelet.key' + "CONTAINER_RUNTIME" = ${kube_env}['CONTAINER_RUNTIME'] + "CONTAINER_RUNTIME_ENDPOINT" = ${kube_env}['CONTAINER_RUNTIME_ENDPOINT'] + # TODO(pjh): these are only in flags, can be removed from env once flags are # moved to util.sh: "LOGS_DIR" = ${kube_env}['LOGS_DIR'] @@ -344,6 +347,7 @@ function DownloadAndInstall-KubernetesBinaries { # Change the directory to the parent directory of ${env:K8S_DIR} and untar. # This (over-)writes ${dest_dir}/kubernetes/node/bin/*.exe files. + # TODO(pjh): clean this up, files not guaranteed to end up in NODE_DIR $dest_dir = (Get-Item ${env:K8S_DIR}).Parent.Fullname tar xzf ${tmp_dir}\${filename} -C ${dest_dir} @@ -836,6 +840,16 @@ Unblock-File $modulePath Import-Module -Name $modulePath'.replace('K8S_DIR', ${env:K8S_DIR}) } +# Setup cni network. This function supports both Docker +# and containerd. +function Configure-CniNetworking { + if (${env:CONTAINER_RUNTIME} -eq "containerd") { + Configure_Containerd_CniNetworking + } else { + Configure_Dockerd_CniNetworking + } +} + # Downloads the Windows CNI binaries and writes a CNI config file under # $env:CNI_CONFIG_DIR. # @@ -850,7 +864,7 @@ Import-Module -Name $modulePath'.replace('K8S_DIR', ${env:K8S_DIR}) # DNS_DOMAIN # CLUSTER_IP_RANGE # SERVICE_CLUSTER_IP_RANGE -function Configure-CniNetworking { +function Configure_Dockerd_CniNetworking { $CNI_RELEASE_VERSION = 'v0.8.2-gke.0' if ((ShouldWrite-File ${env:CNI_DIR}\win-bridge.exe) -or (ShouldWrite-File ${env:CNI_DIR}\host-local.exe)) { @@ -1115,28 +1129,74 @@ function Verify-WorkerServices { Log_Todo "run more verification commands." } +function DownloadAndInstall-Crictl { + $CRICTL_VERSION = "v1.16.1" + $CRICTL_SHA256 = "7d092dcb3b1af2edf75477d5d049a70e8c0d1ac8242b1dff2de7e6aa084e3615" + + # Assume that presence of crictl.exe indicates that the crictl binaries + # were already previously downloaded to this node. + if (-not (ShouldWrite-File ${env:NODE_DIR}\crictl.exe)) { + return + } + $tmp_dir = 'C:\crictl_tmp' + New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null + + $url = ('https://github.com/kubernetes-sigs/cri-tools/releases/' + + 'download/' + $CRICTL_VERSION + '/crictl-' + + $CRICTL_VERSION + '-windows-amd64.tar.gz') + MustDownload-File ` + -URLs $url ` + -OutFile $tmp_dir\crictl.tgz ` + -Hash $CRICTL_SHA256 ` + -Algorithm SHA256 + + Push-Location $tmp_dir + # tar can only extract in the current directory. + tar -xvf $tmp_dir\crictl.tgz + Move-Item -Force crictl.exe ${env:NODE_DIR}\ + if (${env:CONTAINER_RUNTIME_ENDPOINT}) { + crictl.exe config runtime-endpoint ${env:CONTAINER_RUNTIME_ENDPOINT} + } + Pop-Location + Remove-Item -Force -Recurse $tmp_dir +} + # Pulls the infra/pause container image onto the node so that it will be # immediately available when the kubelet tries to run pods. # TODO(pjh): downloading the container container image may take a few minutes; # figure out how to run this in the background while perform the rest of the # node startup steps! +# Pull-InfraContainer must be called AFTER Verify-WorkerServices. function Pull-InfraContainer { $name, $label = $INFRA_CONTAINER -split ':',2 - if (-not ("$(& docker image list)" -match "$name.*$label")) { - & docker pull $INFRA_CONTAINER + if (-not ("$(& crictl images)" -match "$name.*$label")) { + & crictl pull $INFRA_CONTAINER if (!$?) { - throw "Error running 'docker pull $INFRA_CONTAINER'" + throw "Error running 'crictl pull $INFRA_CONTAINER'" } } - $inspect = "$(& docker inspect $INFRA_CONTAINER | Out-String)" + $inspect = "$(& crictl inspecti $INFRA_CONTAINER | Out-String)" Log-Output "Infra/pause container:`n$inspect" } +# Setup the container runtime on the node. It supports both +# Docker and containerd. +function Setup-ContainerRuntime { + if (${env:CONTAINER_RUNTIME} -eq "containerd") { + Install_Containerd + Start_Containerd + } else { + Create_DockerRegistryKey + Configure_Dockerd + } +} + # Add a registry key for docker in EventLog so that log messages are mapped # correctly. This is a workaround since the key is missing in the base image. # https://github.com/MicrosoftDocs/Virtualization-Documentation/pull/503 # TODO: Fix this in the base image. -function Create-DockerRegistryKey { +# TODO(random-liu): Figure out whether we need this for containerd. +function Create_DockerRegistryKey { $tmp_dir = 'C:\tmp_docker_reg' New-Item -Force -ItemType 'directory' ${tmp_dir} | Out-Null $reg_file = 'docker.reg' @@ -1153,7 +1213,7 @@ function Create-DockerRegistryKey { } # Configure Docker daemon and restart the service. -function Configure-Dockerd { +function Configure_Dockerd { Set-Content "C:\ProgramData\docker\config\daemon.json" @' { "log-driver": "json-file", @@ -1167,6 +1227,180 @@ function Configure-Dockerd { Restart-Service Docker } +# Writes a CNI config file under $env:CNI_CONFIG_DIR for containerd. +# +# Prerequisites: +# $env:POD_CIDR is set (by Set-PodCidr). +# The "management" interface exists (Configure-HostNetworkingService). +# The HNS network for pod networking has been configured +# (Configure-HostNetworkingService). +# Containerd is installed (Install_Containerd). +# +# Required ${kube_env} keys: +# DNS_SERVER_IP +# DNS_DOMAIN +# CLUSTER_IP_RANGE +# SERVICE_CLUSTER_IP_RANGE +function Configure_Containerd_CniNetworking { + $l2bridge_conf = "${env:CNI_CONFIG_DIR}\l2bridge.conf" + if (-not (ShouldWrite-File ${l2bridge_conf})) { + return + } + + $mgmt_ip = (Get_MgmtNetAdapter | + Get-NetIPAddress -AddressFamily IPv4).IPAddress + $mgmt_subnet = Get_MgmtSubnet + Log-Output ("using mgmt IP ${mgmt_ip} and mgmt subnet ${mgmt_subnet} for " + + "CNI config") + + $pod_gateway = Get_Endpoint_Gateway_From_CIDR(${env:POD_CIDR}) + + # Explanation of the CNI config values: + # CLUSTER_CIDR: the cluster CIDR from which pod CIDRs are allocated. + # POD_CIDR: the pod CIDR assigned to this node. + # POD_GATEWAY: the gateway IP. + # MGMT_SUBNET: the subnet on which the Windows pods + kubelet will + # communicate with the rest of the cluster without NAT (i.e. the subnet + # that VM internal IPs are allocated from). + # MGMT_IP: the IP address assigned to the node's primary network interface + # (i.e. the internal IP of the GCE VM). + # SERVICE_CIDR: the CIDR used for kubernetes services. + # DNS_SERVER_IP: the cluster's DNS server IP address. + # DNS_DOMAIN: the cluster's DNS domain, e.g. "cluster.local". + New-Item -Force -ItemType file ${l2bridge_conf} | Out-Null + Set-Content ${l2bridge_conf} ` +'{ + "cniVersion": "0.2.0", + "name": "l2bridge", + "type": "sdnbridge", + "master": "Ethernet", + "capabilities": { + "portMappings": true, + "dns": true + }, + "ipam": { + "subnet": "POD_CIDR", + "routes": [ + { + "GW": "POD_GATEWAY" + } + ] + }, + "dns": { + "Nameservers": [ + "DNS_SERVER_IP" + ], + "Search": [ + "DNS_DOMAIN" + ] + }, + "AdditionalArgs": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "Settings": { + "Exceptions": [ + "CLUSTER_CIDR", + "SERVICE_CIDR", + "MGMT_SUBNET" + ] + } + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "SDNRoute", + "Settings": { + "DestinationPrefix": "SERVICE_CIDR", + "NeedEncap": true + } + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "SDNRoute", + "Settings": { + "DestinationPrefix": "MGMT_IP/32", + "NeedEncap": true + } + } + } + ] +}'.replace('POD_CIDR', ${env:POD_CIDR}).` + replace('POD_GATEWAY', ${pod_gateway}).` + replace('DNS_SERVER_IP', ${kube_env}['DNS_SERVER_IP']).` + replace('DNS_DOMAIN', ${kube_env}['DNS_DOMAIN']).` + replace('MGMT_IP', ${mgmt_ip}).` + replace('CLUSTER_CIDR', ${kube_env}['CLUSTER_IP_RANGE']).` + replace('SERVICE_CIDR', ${kube_env}['SERVICE_CLUSTER_IP_RANGE']).` + replace('MGMT_SUBNET', ${mgmt_subnet}) + + Log-Output "CNI config:`n$(Get-Content -Raw ${l2bridge_conf})" +} + +# Download and install containerd and CNI binaries. +function Install_Containerd { + # Assume that presence of containerd.exe indicates that all containerd binaries + # were already previously downloaded to this node. + if (-not (ShouldWrite-File ${env:NODE_DIR}\containerd.exe)) { + return + } + + # https://storage.googleapis.com/cri-containerd-staging/cri-containerd-9f79be1b.windows-amd64.tar.gz + # TODO(random-liu): Change this to official release path after testing. + $CONTAINERD_GCS_BUCKET = "cri-containerd-staging/windows" + + $tmp_dir = 'C:\containerd_tmp' + New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null + + $version_url = "https://storage.googleapis.com/$CONTAINERD_GCS_BUCKET/latest" + MustDownload-File -URLs $version_url -OutFile $tmp_dir\version + $version = $(Get-Content $tmp_dir\version) + + $tar_url = "https://storage.googleapis.com/$CONTAINERD_GCS_BUCKET/cri-containerd-cni-$version.windows-amd64.tar.gz" + $sha_url = $tar_url + ".sha256" + MustDownload-File -URLs $sha_url -OutFile $tmp_dir\sha256 + $sha = $(Get-Content $tmp_dir\sha256) + + MustDownload-File ` + -URLs $tar_url ` + -OutFile $tmp_dir\containerd.tar.gz ` + -Hash $sha ` + -Algorithm SHA256 + + Push-Location $tmp_dir + # tar can only extract in the current directory. + tar -xvf $tmp_dir\containerd.tar.gz + Move-Item -Force cni\*.exe ${env:CNI_DIR}\ + Move-Item -Force *.exe ${env:NODE_DIR}\ + Pop-Location + Remove-Item -Force -Recurse $tmp_dir + + # Generate containerd config + $config_dir = 'C:\Program Files\containerd' + New-Item $config_dir -ItemType 'directory' -Force | Out-Null + Set-Content "$config_dir\config.toml" @" +[plugins.cri] + sandbox_image = 'INFRA_CONTAINER_IMAGE' +[plugins.cri.cni] + bin_dir = 'CNI_BIN_DIR' + conf_dir = 'CNI_CONF_DIR' +"@.replace('INFRA_CONTAINER_IMAGE', $INFRA_CONTAINER).` + replace('CNI_BIN_DIR', ${env:CNI_DIR}).` + replace('CNI_CONF_DIR', ${env:CNI_CONFIG_DIR}) +} + +# Register and start containerd service. +function Start_Containerd { + Log-Output "Creating containerd service" + containerd.exe --register-service + Log-Output "Starting containerd service" + Start-Service containerd +} + # TODO(pjh): move the Stackdriver logging agent code below into a separate # module; it was put here temporarily to avoid disrupting the file layout in # the K8s release machinery. @@ -1261,6 +1495,11 @@ function Install-LoggingAgent { -ArgumentList "install","fluent-plugin-record-reformer" ` -Wait + # Install the multi-format-parser plugin. + Start-Process "$STACKDRIVER_ROOT\LoggingAgent\Main\bin\fluent-gem" ` + -ArgumentList "install","fluent-plugin-multi-format-parser" ` + -Wait + Remove-Item -Force -Recurse $tmp_dir } @@ -1329,7 +1568,8 @@ $FLUENTD_CONFIG = @' # Json Log Example: # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} -# TODO: Support CRI log format, which requires the multi_format plugin. +# CRI Log Example: +# 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here @type tail path /var/log/containers/*.log @@ -1337,10 +1577,19 @@ $FLUENTD_CONFIG = @' # Tags at this point are in the format of: # reform.var.log.containers.__-.log tag reform.* - format json - time_key time - time_format %Y-%m-%dT%H:%M:%S.%NZ read_from_head true + + @type multi_format + + format json + time_key time + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format /^(? + # Example: