Merge pull request #87907 from pjh/configure-windows-gce-carefully

Separate containerd install from config, and other cleanups
This commit is contained in:
Kubernetes Prow Robot 2020-02-13 16:55:01 -08:00 committed by GitHub
commit 461a494f69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 96 additions and 115 deletions

View File

@ -132,6 +132,7 @@ try {
Download-HelperScripts
DownloadAndInstall-Crictl
Configure-Crictl
Setup-ContainerRuntime
DownloadAndInstall-AuthPlugin
DownloadAndInstall-KubernetesBinaries
@ -140,7 +141,7 @@ try {
Create-KubeproxyKubeconfig
Set-PodCidr
Configure-HostNetworkingService
Configure-CniNetworking
Prepare-CniNetworking
Configure-HostDnsConf
Configure-GcePdTools
Configure-Kubelet

View File

@ -58,6 +58,8 @@ $GCE_METADATA_SERVER = "169.254.169.254"
# exist until an initial HNS network has been created on the Windows node - see
# Add_InitialHnsNetwork().
$MGMT_ADAPTER_NAME = "vEthernet (Ethernet*"
$CRICTL_VERSION = 'v1.17.0'
$CRICTL_SHA256 = '781fd3bd15146a924c6fc2428b11d8a0f20fa04a0c8e00a9a5808f2cc37e0569'
Import-Module -Force C:\common.psm1
@ -151,7 +153,7 @@ function Dump-DebugInfoToConsole {
#
# Returns: a PowerShell Hashtable object containing the key-value pairs from
# kube-env.
function ConvertFrom-Yaml-KubeEnv {
function ConvertFrom_Yaml_KubeEnv {
param (
[parameter(Mandatory=$true)] [string]$kube_env_str
)
@ -194,13 +196,13 @@ function Fetch-KubeEnv {
# ${kube_env} = Get-InstanceMetadataAttribute 'kube-env'
# or:
# ${kube_env} = [IO.File]::ReadAllText(".\kubeEnv.txt")
# ${kube_env_table} = ConvertFrom-Yaml-KubeEnv ${kube_env}
# ${kube_env_table} = ConvertFrom_Yaml_KubeEnv ${kube_env}
# ${kube_env_table}
# ${kube_env_table}.GetType()
# The type of kube_env is a powershell String.
$kube_env = Get-InstanceMetadataAttribute 'kube-env'
$kube_env_table = ConvertFrom-Yaml-KubeEnv ${kube_env}
$kube_env_table = ConvertFrom_Yaml_KubeEnv ${kube_env}
return ${kube_env_table}
}
@ -233,6 +235,7 @@ function Set-EnvironmentVars {
# ${kube_env}['K8S_DIR'] cannot be afaik).
$env_vars = @{
"K8S_DIR" = ${kube_env}['K8S_DIR']
# Typically 'C:\etc\kubernetes\node\bin' (not just 'C:\etc\kubernetes\node')
"NODE_DIR" = ${kube_env}['NODE_DIR']
"CNI_DIR" = ${kube_env}['CNI_DIR']
"CNI_CONFIG_DIR" = ${kube_env}['CNI_CONFIG_DIR']
@ -273,8 +276,8 @@ function Set-EnvironmentVars {
function Set-PrerequisiteOptions {
# Windows updates cause the node to reboot at arbitrary times.
Log-Output "Disabling Windows Update service"
sc.exe config wuauserv start=disabled
sc.exe stop wuauserv
& sc.exe config wuauserv start=disabled
& sc.exe stop wuauserv
# Use TLS 1.2: needed for Invoke-WebRequest downloads from github.com.
[Net.ServicePointManager]::SecurityProtocol = `
@ -298,29 +301,11 @@ function Create-Directories {
# Downloads some external helper scripts needed by other functions in this
# module.
function Download-HelperScripts {
if (-not (ShouldWrite-File ${env:K8S_DIR}\hns.psm1)) {
return
if (ShouldWrite-File ${env:K8S_DIR}\hns.psm1) {
MustDownload-File `
-OutFile ${env:K8S_DIR}\hns.psm1 `
-URLs 'https://storage.googleapis.com/gke-release/winnode/config/sdn/master/hns.psm1'
}
MustDownload-File -OutFile ${env:K8S_DIR}\hns.psm1 `
-URLs "https://storage.googleapis.com/gke-release/winnode/config/sdn/master/hns.psm1"
}
# Takes the Windows version string from the cluster bash scripts (e.g.
# 'win1809') and returns the correct label to use for containers on this
# version of Windows. Returns $null if $WinVersion is unknown.
function Get_ContainerVersionLabel {
param (
[parameter(Mandatory=$true)] [string]$WinVersion
)
# -match does regular expression matching.
if ($WinVersion -match '1809') {
return '1809'
}
elseif ($WinVersion -match '2019') {
return 'ltsc2019'
}
Throw ("Unknown Windows version $WinVersion, don't know its container " +
"version label")
}
# Downloads the gke-exec-auth-plugin for TPM-based authentication to the
@ -377,13 +362,12 @@ function DownloadAndInstall-KubernetesBinaries {
if ($kube_env.ContainsKey('NODE_BINARY_TAR_HASH')) {
$hash = ${kube_env}['NODE_BINARY_TAR_HASH']
}
MustDownload-File -Hash $hash -OutFile ${tmp_dir}\${filename} -URLs $urls
MustDownload-File -Hash $hash -OutFile $tmp_dir\$filename -URLs $urls
# Change the directory to the parent directory of ${env:K8S_DIR} and untar.
# This (over-)writes ${dest_dir}/kubernetes/node/bin/*.exe files.
# TODO(pjh): clean this up, files not guaranteed to end up in NODE_DIR
$dest_dir = (Get-Item ${env:K8S_DIR}).Parent.Fullname
tar xzf ${tmp_dir}\${filename} -C ${dest_dir}
tar xzvf $tmp_dir\$filename -C $tmp_dir
Move-Item -Force $tmp_dir\kubernetes\node\bin\* ${env:NODE_DIR}\
Move-Item -Force `
$tmp_dir\kubernetes\LICENSES ${env:LICENSE_DIR}\LICENSES_kubernetes
# Clean up the temporary directory
Remove-Item -Force -Recurse $tmp_dir
@ -924,24 +908,59 @@ function Configure-GcePdTools {
New-Item -path $PsHome\profile.ps1 -type file
}
Add-Content $PsHome\profile.ps1 `
Add-Content $PsHome\profile.ps1 `
'$modulePath = "K8S_DIR\GetGcePdName.dll"
Unblock-File $modulePath
Import-Module -Name $modulePath'.replace('K8S_DIR', ${env:K8S_DIR})
}
# Setup cni network. This function supports both Docker
# and containerd.
function Configure-CniNetworking {
# Setup cni network. This function supports both Docker and containerd.
function Prepare-CniNetworking {
if (${env:CONTAINER_RUNTIME} -eq "containerd") {
# For containerd the CNI binaries have already been installed along with
# the runtime.
Configure_Containerd_CniNetworking
} else {
Install_Cni_Binaries
Configure_Dockerd_CniNetworking
}
}
# Downloads the Windows CNI binaries and writes a CNI config file under
# $env:CNI_CONFIG_DIR.
# Downloads the Windows CNI binaries and puts them in $env:CNI_DIR.
function Install_Cni_Binaries {
if (-not (ShouldWrite-File ${env:CNI_DIR}\win-bridge.exe) -and
-not (ShouldWrite-File ${env:CNI_DIR}\host-local.exe)) {
return
}
$tmp_dir = 'C:\cni_tmp'
New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null
$release_url = "${env:WINDOWS_CNI_STORAGE_PATH}/${env:WINDOWS_CNI_VERSION}/"
$tgz_url = ($release_url +
"cni-plugins-windows-amd64-${env:WINDOWS_CNI_VERSION}.tgz")
$sha_url = ($tgz_url + ".sha1")
MustDownload-File -URLs $sha_url -OutFile $tmp_dir\cni-plugins.sha1
$sha1_val = ($(Get-Content $tmp_dir\cni-plugins.sha1) -split ' ',2)[0]
MustDownload-File `
-URLs $tgz_url `
-OutFile $tmp_dir\cni-plugins.tgz `
-Hash $sha1_val
tar xzvf $tmp_dir\cni-plugins.tgz -C $tmp_dir
Move-Item -Force $tmp_dir\host-local.exe ${env:CNI_DIR}\
Move-Item -Force $tmp_dir\win-bridge.exe ${env:CNI_DIR}\
Remove-Item -Force -Recurse $tmp_dir
if (-not ((Test-Path ${env:CNI_DIR}\win-bridge.exe) -and `
(Test-Path ${env:CNI_DIR}\host-local.exe))) {
Log-Output `
"win-bridge.exe and host-local.exe not found in ${env:CNI_DIR}" `
-Fatal
}
}
# Writes a CNI config file under $env:CNI_CONFIG_DIR.
#
# Prerequisites:
# $env:POD_CIDR is set (by Set-PodCidr).
@ -955,37 +974,6 @@ function Configure-CniNetworking {
# CLUSTER_IP_RANGE
# SERVICE_CLUSTER_IP_RANGE
function Configure_Dockerd_CniNetworking {
if ((ShouldWrite-File ${env:CNI_DIR}\win-bridge.exe) -or
(ShouldWrite-File ${env:CNI_DIR}\host-local.exe)) {
$tmp_dir = 'C:\cni_tmp'
New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null
$release_url = (${env:WINDOWS_CNI_STORAGE_PATH} + '/' + ${env:WINDOWS_CNI_VERSION} + '/')
$tgz_url = ($release_url +
"cni-plugins-windows-amd64-${env:WINDOWS_CNI_VERSION}.tgz")
$sha_url = ($tgz_url + ".sha1")
MustDownload-File -URLs $sha_url -OutFile $tmp_dir\cni-plugins.sha1
$sha1_val = ($(Get-Content $tmp_dir\cni-plugins.sha1) -split ' ',2)[0]
MustDownload-File `
-URLs $tgz_url `
-OutFile $tmp_dir\cni-plugins.tgz `
-Hash $sha1_val
Push-Location $tmp_dir
# tar can only extract in the current directory.
tar -xvf $tmp_dir\cni-plugins.tgz
Move-Item -Force host-local.exe ${env:CNI_DIR}\
Move-Item -Force win-bridge.exe ${env:CNI_DIR}\
Pop-Location
Remove-Item -Force -Recurse $tmp_dir
}
if (-not ((Test-Path ${env:CNI_DIR}\win-bridge.exe) -and `
(Test-Path ${env:CNI_DIR}\host-local.exe))) {
Log-Output `
"win-bridge.exe and host-local.exe not found in ${env:CNI_DIR}" `
-Fatal
}
$l2bridge_conf = "${env:CNI_CONFIG_DIR}\l2bridge.conf"
if (-not (ShouldWrite-File ${l2bridge_conf})) {
return
@ -1088,7 +1076,7 @@ function Configure-HostDnsConf {
$search_list = (Get-DnsClient).ConnectionSpecificSuffixSearchList
$conf = ""
ForEach ($ip in $server_ips) {
$conf = $conf + "nameserver $ip`r`n"
$conf = $conf + "nameserver $ip`r`n"
}
$conf = $conf + "search $search_list"
# Do not put hostdns.conf into the CNI config directory so as to
@ -1182,10 +1170,10 @@ function Start-WorkerServices {
"A kubelet process is already running, don't know what to do"
}
Log-Output "Creating kubelet service"
sc.exe create kubelet binPath= "${env:NODE_DIR}\kubelet.exe ${kubelet_args}" start= demand
sc.exe failure kubelet reset= 0 actions= restart/10000
& sc.exe create kubelet binPath= "${env:NODE_DIR}\kubelet.exe ${kubelet_args}" start= demand
& sc.exe failure kubelet reset= 0 actions= restart/10000
Log-Output "Starting kubelet service"
sc.exe start kubelet
& sc.exe start kubelet
Log-Output "Waiting 10 seconds for kubelet to stabilize"
Start-Sleep 10
@ -1195,10 +1183,10 @@ function Start-WorkerServices {
"A kube-proxy process is already running, don't know what to do"
}
Log-Output "Creating kube-proxy service"
sc.exe create kube-proxy binPath= "${env:NODE_DIR}\kube-proxy.exe ${kubeproxy_args}" start= demand
sc.exe failure kube-proxy reset= 0 actions= restart/10000
& sc.exe create kube-proxy binPath= "${env:NODE_DIR}\kube-proxy.exe ${kubeproxy_args}" start= demand
& sc.exe failure kube-proxy reset= 0 actions= restart/10000
Log-Output "Starting kube-proxy service"
sc.exe start kube-proxy
& sc.exe start kube-proxy
# F1020 23:08:52.000083 9136 server.go:361] unable to load in-cluster
# configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be
@ -1211,7 +1199,7 @@ function Start-WorkerServices {
Log-Output "Kubernetes components started successfully"
}
# Wait for kubelet & kube-proxy to be ready within 10s.
# Wait for kubelet and kube-proxy to be ready within 10s.
function WaitFor_KubeletAndKubeProxyReady {
$waited = 0
$timeout = 10
@ -1223,7 +1211,7 @@ function WaitFor_KubeletAndKubeProxyReady {
# Timeout occurred
if ($waited -ge $timeout) {
Log-Output "$(Get-Service kube* | Out-String)"
Throw ("Timeout while waiting ${timeout} seconds for kubelet & kube-proxy services to start")
Throw ("Timeout while waiting ${timeout} seconds for kubelet and kube-proxy services to start")
}
}
@ -1231,38 +1219,31 @@ function WaitFor_KubeletAndKubeProxyReady {
# TODO(pjh): run more verification commands.
function Verify-WorkerServices {
Log-Output ("kubectl get nodes:`n" +
"$(& ${env:NODE_DIR}\kubectl.exe get nodes | Out-String)")
$(& "${env:NODE_DIR}\kubectl.exe" get nodes | Out-String))
Verify_GceMetadataServerRouteIsPresent
Log_Todo "run more verification commands."
}
# Downloads crictl.exe and installs it in $env:NODE_DIR.
function DownloadAndInstall-Crictl {
$CRICTL_VERSION = "v1.17.0"
$CRICTL_SHA256 = "781fd3bd15146a924c6fc2428b11d8a0f20fa04a0c8e00a9a5808f2cc37e0569"
# Assume that presence of crictl.exe indicates that the crictl binaries
# were already previously downloaded to this node.
if (-not (ShouldWrite-File ${env:NODE_DIR}\crictl.exe)) {
return
}
$tmp_dir = 'C:\crictl_tmp'
New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null
$url = ('https://storage.googleapis.com/kubernetes-release/crictl/' +
'crictl-' + $CRICTL_VERSION + '-windows-amd64.exe')
MustDownload-File `
-URLs $url `
-OutFile $tmp_dir\crictl.exe `
-OutFile ${env:NODE_DIR}\crictl.exe `
-Hash $CRICTL_SHA256 `
-Algorithm SHA256
}
Push-Location $tmp_dir
Move-Item -Force crictl.exe ${env:NODE_DIR}\
# Sets crictl configuration values.
function Configure-Crictl {
if (${env:CONTAINER_RUNTIME_ENDPOINT}) {
crictl.exe config runtime-endpoint ${env:CONTAINER_RUNTIME_ENDPOINT}
& "${env:NODE_DIR}\crictl.exe" config runtime-endpoint `
${env:CONTAINER_RUNTIME_ENDPOINT}
}
Pop-Location
Remove-Item -Force -Recurse $tmp_dir
}
# Pulls the infra/pause container image onto the node so that it will be
@ -1288,6 +1269,7 @@ function Pull-InfraContainer {
function Setup-ContainerRuntime {
if (${env:CONTAINER_RUNTIME} -eq "containerd") {
Install_Containerd
Configure_Containerd
Start_Containerd
} else {
Create_DockerRegistryKey
@ -1403,7 +1385,7 @@ function Configure_Containerd_CniNetworking {
"Name": "EndpointPolicy",
"Value": {
"Type": "OutBoundNAT",
"Settings": {
"Settings": {
"Exceptions": [
"CLUSTER_CIDR",
"SERVICE_CIDR",
@ -1416,20 +1398,20 @@ function Configure_Containerd_CniNetworking {
"Name": "EndpointPolicy",
"Value": {
"Type": "SDNRoute",
"Settings": {
"Settings": {
"DestinationPrefix": "SERVICE_CIDR",
"NeedEncap": true
}
}
}
},
{
"Name": "EndpointPolicy",
"Value": {
"Type": "SDNRoute",
"Settings": {
"Settings": {
"DestinationPrefix": "MGMT_IP/32",
"NeedEncap": true
}
}
}
}
]
@ -1442,18 +1424,17 @@ function Configure_Containerd_CniNetworking {
replace('SERVICE_CIDR', ${kube_env}['SERVICE_CLUSTER_IP_RANGE']).`
replace('MGMT_SUBNET', ${mgmt_subnet})
Log-Output "CNI config:`n$(Get-Content -Raw ${l2bridge_conf})"
Log-Output "containerd CNI config:`n$(Get-Content -Raw ${l2bridge_conf})"
}
# Download and install containerd and CNI binaries.
# Download and install containerd and CNI binaries into $env:NODE_DIR.
function Install_Containerd {
# Assume that presence of containerd.exe indicates that all containerd binaries
# were already previously downloaded to this node.
# Assume that presence of containerd.exe indicates that all containerd
# binaries were already previously downloaded to this node.
if (-not (ShouldWrite-File ${env:NODE_DIR}\containerd.exe)) {
return
}
# https://storage.googleapis.com/cri-containerd-staging/cri-containerd-9f79be1b.windows-amd64.tar.gz
# TODO(random-liu): Change this to official release path after testing.
$CONTAINERD_GCS_BUCKET = "cri-containerd-staging/windows"
@ -1464,7 +1445,8 @@ function Install_Containerd {
MustDownload-File -URLs $version_url -OutFile $tmp_dir\version
$version = $(Get-Content $tmp_dir\version)
$tar_url = "https://storage.googleapis.com/$CONTAINERD_GCS_BUCKET/cri-containerd-cni-$version.windows-amd64.tar.gz"
$tar_url = ("https://storage.googleapis.com/$CONTAINERD_GCS_BUCKET/" +
"cri-containerd-cni-$version.windows-amd64.tar.gz")
$sha_url = $tar_url + ".sha256"
MustDownload-File -URLs $sha_url -OutFile $tmp_dir\sha256
$sha = $(Get-Content $tmp_dir\sha256)
@ -1475,15 +1457,14 @@ function Install_Containerd {
-Hash $sha `
-Algorithm SHA256
Push-Location $tmp_dir
# tar can only extract in the current directory.
tar -xvf $tmp_dir\containerd.tar.gz
Move-Item -Force cni\*.exe ${env:CNI_DIR}\
Move-Item -Force *.exe ${env:NODE_DIR}\
Pop-Location
tar xzvf $tmp_dir\containerd.tar.gz -C $tmp_dir
Move-Item -Force $tmp_dir\cni\*.exe ${env:CNI_DIR}\
Move-Item -Force $tmp_dir\*.exe ${env:NODE_DIR}\
Remove-Item -Force -Recurse $tmp_dir
}
# Generate containerd config
# Generates the containerd config.toml file.
function Configure_Containerd {
$config_dir = 'C:\Program Files\containerd'
New-Item $config_dir -ItemType 'directory' -Force | Out-Null
Set-Content "$config_dir\config.toml" @"
@ -1500,7 +1481,7 @@ function Install_Containerd {
# Register and start containerd service.
function Start_Containerd {
Log-Output "Creating containerd service"
containerd.exe --register-service --log-file ${env:LOGS_DIR}/containerd.log
& containerd.exe --register-service --log-file ${env:LOGS_DIR}/containerd.log
Log-Output "Starting containerd service"
Start-Service containerd
}
@ -1511,7 +1492,6 @@ function Start_Containerd {
$STACKDRIVER_VERSION = 'v1-9'
$STACKDRIVER_ROOT = 'C:\Program Files (x86)\Stackdriver'
# Restarts the Stackdriver logging agent, or starts it if it is not currently
# running. A standard `Restart-Service StackdriverLogging` may fail because
# StackdriverLogging sometimes is unstoppable, so this function works around it