mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #85466 from pjh/gce-shielded-windows-master
Update GCE Windows startup scripts for plugin-based authentication
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
d9ed6110e8
@ -144,6 +144,8 @@ export WINDOWS_CNI_CONFIG_DIR="${WINDOWS_K8S_DIR}\cni\config"
|
||||
export WINDOWS_MANIFESTS_DIR="${WINDOWS_K8S_DIR}\manifests"
|
||||
# Directory where cert/key files will be stores on Windows nodes.
|
||||
export WINDOWS_PKI_DIR="${WINDOWS_K8S_DIR}\pki"
|
||||
# Location of the certificates file on Windows nodes.
|
||||
export WINDOWS_CA_FILE="${WINDOWS_PKI_DIR}\ca-certificates.crt"
|
||||
# Path for kubelet config file on Windows nodes.
|
||||
export WINDOWS_KUBELET_CONFIG_FILE="${WINDOWS_K8S_DIR}\kubelet-config.yaml"
|
||||
# Path for kubeconfig file on Windows nodes.
|
||||
|
||||
@ -839,17 +839,6 @@ function construct-windows-kubelet-flags {
|
||||
# Many of these flags were adapted from
|
||||
# https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1.
|
||||
flags+=" --config=${WINDOWS_KUBELET_CONFIG_FILE}"
|
||||
|
||||
# Path to a kubeconfig file that will be used to get client certificate for
|
||||
# kubelet. If the file specified by --kubeconfig does not exist, the bootstrap
|
||||
# kubeconfig is used to request a client certificate from the API server. On
|
||||
# success, a kubeconfig file referencing the generated client certificate and
|
||||
# key is written to the path specified by --kubeconfig. The client certificate
|
||||
# and key file will be stored in the directory pointed by --cert-dir.
|
||||
#
|
||||
# See also:
|
||||
# https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/
|
||||
flags+=" --bootstrap-kubeconfig=${WINDOWS_BOOTSTRAP_KUBECONFIG_FILE}"
|
||||
flags+=" --kubeconfig=${WINDOWS_KUBECONFIG_FILE}"
|
||||
|
||||
# The directory where the TLS certs are located.
|
||||
@ -1066,7 +1055,7 @@ function print-windows-node-kubelet-config {
|
||||
cat <<EOF
|
||||
authentication:
|
||||
x509:
|
||||
clientCAFile: '${WINDOWS_PKI_DIR}\ca-certificates.crt'
|
||||
clientCAFile: '${WINDOWS_CA_FILE}'
|
||||
EOF
|
||||
}
|
||||
|
||||
@ -1534,6 +1523,7 @@ CNI_DIR: $(yaml-quote ${WINDOWS_CNI_DIR})
|
||||
CNI_CONFIG_DIR: $(yaml-quote ${WINDOWS_CNI_CONFIG_DIR})
|
||||
MANIFESTS_DIR: $(yaml-quote ${WINDOWS_MANIFESTS_DIR})
|
||||
PKI_DIR: $(yaml-quote ${WINDOWS_PKI_DIR})
|
||||
CA_FILE_PATH: $(yaml-quote ${WINDOWS_CA_FILE})
|
||||
KUBELET_CONFIG_FILE: $(yaml-quote ${WINDOWS_KUBELET_CONFIG_FILE})
|
||||
KUBEPROXY_ARGS: $(yaml-quote ${KUBEPROXY_ARGS})
|
||||
KUBECONFIG_FILE: $(yaml-quote ${WINDOWS_KUBECONFIG_FILE})
|
||||
|
||||
@ -19,8 +19,9 @@
|
||||
#>
|
||||
|
||||
# IMPORTANT PLEASE NOTE:
|
||||
# Any time the file structure in the `windows` directory changes, `windows/BUILD`
|
||||
# and `k8s.io/release/lib/releaselib.sh` must be manually updated with the changes.
|
||||
# Any time the file structure in the `windows` directory changes,
|
||||
# `windows/BUILD` and `k8s.io/release/lib/releaselib.sh` must be manually
|
||||
# updated with the changes.
|
||||
# We HIGHLY recommend not changing the file structure, because consumers of
|
||||
# Kubernetes releases depend on the release structure remaining stable.
|
||||
|
||||
@ -544,5 +545,16 @@ function Test-IsTestCluster {
|
||||
return $false
|
||||
}
|
||||
|
||||
# Returns true if this node uses a plugin to support authentication to the
|
||||
# master, e.g. for TPM-based authentication. $KubeEnv is a hash table
|
||||
# containing the kube-env metadata keys+values.
|
||||
function Test-NodeUsesAuthPlugin {
|
||||
param (
|
||||
[parameter(Mandatory=$true)] [hashtable]$KubeEnv
|
||||
)
|
||||
|
||||
return $KubeEnv.Contains('EXEC_AUTH_PLUGIN_URL')
|
||||
}
|
||||
|
||||
# Export all public functions:
|
||||
Export-ModuleMember -Function *-*
|
||||
|
||||
@ -133,6 +133,7 @@ try {
|
||||
|
||||
DownloadAndInstall-Crictl
|
||||
Setup-ContainerRuntime
|
||||
DownloadAndInstall-AuthPlugin
|
||||
DownloadAndInstall-KubernetesBinaries
|
||||
Create-NodePki
|
||||
Create-KubeletKubeconfig
|
||||
|
||||
@ -237,24 +237,23 @@ function Set-EnvironmentVars {
|
||||
"CNI_DIR" = ${kube_env}['CNI_DIR']
|
||||
"CNI_CONFIG_DIR" = ${kube_env}['CNI_CONFIG_DIR']
|
||||
"PKI_DIR" = ${kube_env}['PKI_DIR']
|
||||
"CA_FILE_PATH" = ${kube_env}['CA_FILE_PATH']
|
||||
"KUBELET_CONFIG" = ${kube_env}['KUBELET_CONFIG_FILE']
|
||||
"BOOTSTRAP_KUBECONFIG" = ${kube_env}['BOOTSTRAP_KUBECONFIG_FILE']
|
||||
"KUBECONFIG" = ${kube_env}['KUBECONFIG_FILE']
|
||||
"KUBEPROXY_KUBECONFIG" = ${kube_env}['KUBEPROXY_KUBECONFIG_FILE']
|
||||
"LOGS_DIR" = ${kube_env}['LOGS_DIR']
|
||||
"MANIFESTS_DIR" = ${kube_env}['MANIFESTS_DIR']
|
||||
|
||||
"Path" = ${env:Path} + ";" + ${kube_env}['NODE_DIR']
|
||||
"KUBE_NETWORK" = "l2bridge".ToLower()
|
||||
"CA_CERT_BUNDLE_PATH" = ${kube_env}['PKI_DIR'] + '\ca-certificates.crt'
|
||||
"KUBELET_CERT_PATH" = ${kube_env}['PKI_DIR'] + '\kubelet.crt'
|
||||
"KUBELET_KEY_PATH" = ${kube_env}['PKI_DIR'] + '\kubelet.key'
|
||||
|
||||
"CONTAINER_RUNTIME" = ${kube_env}['CONTAINER_RUNTIME']
|
||||
"CONTAINER_RUNTIME_ENDPOINT" = ${kube_env}['CONTAINER_RUNTIME_ENDPOINT']
|
||||
|
||||
# TODO(pjh): these are only in flags, can be removed from env once flags are
|
||||
# moved to util.sh:
|
||||
"LOGS_DIR" = ${kube_env}['LOGS_DIR']
|
||||
"MANIFESTS_DIR" = ${kube_env}['MANIFESTS_DIR']
|
||||
"KUBECONFIG" = ${kube_env}['KUBECONFIG_FILE']
|
||||
'LICENSE_DIR' = 'C:\Program Files\Google\Compute Engine\THIRD_PARTY_NOTICES'
|
||||
}
|
||||
|
||||
# Set the environment variables in two ways: permanently on the machine (only
|
||||
@ -289,7 +288,7 @@ function Create-Directories {
|
||||
Log-Output "Creating ${env:K8S_DIR} and its subdirectories."
|
||||
ForEach ($dir in ("${env:K8S_DIR}", "${env:NODE_DIR}", "${env:LOGS_DIR}",
|
||||
"${env:CNI_DIR}", "${env:CNI_CONFIG_DIR}", "${env:MANIFESTS_DIR}",
|
||||
"${env:PKI_DIR}"), "C:\tmp", "C:\var\log") {
|
||||
"${env:PKI_DIR}", "${env:LICENSE_DIR}"), "C:\tmp", "C:\var\log") {
|
||||
mkdir -Force $dir
|
||||
}
|
||||
}
|
||||
@ -322,6 +321,39 @@ function Get_ContainerVersionLabel {
|
||||
"version label")
|
||||
}
|
||||
|
||||
# Downloads the gke-exec-auth-plugin for TPM-based authentication to the
|
||||
# master, if auth plugin support has been requested for this node (see
|
||||
# Test-NodeUsesAuthPlugin).
|
||||
# https://github.com/kubernetes/cloud-provider-gcp/tree/master/cmd/gke-exec-auth-plugin
|
||||
#
|
||||
# Required ${kube_env} keys:
|
||||
# EXEC_AUTH_PLUGIN_LICENSE_URL
|
||||
# EXEC_AUTH_PLUGIN_SHA1
|
||||
# EXEC_AUTH_PLUGIN_URL
|
||||
function DownloadAndInstall-AuthPlugin {
|
||||
if (-not (Test-NodeUsesAuthPlugin ${kube_env})) {
|
||||
Log-Output 'Skipping download of auth plugin'
|
||||
return
|
||||
}
|
||||
if (-not (ShouldWrite-File "${env:NODE_DIR}\gke-exec-auth-plugin.exe")) {
|
||||
return
|
||||
}
|
||||
|
||||
if (-not ($kube_env.ContainsKey('EXEC_AUTH_PLUGIN_LICENSE_URL') -and
|
||||
$kube_env.ContainsKey('EXEC_AUTH_PLUGIN_SHA1') -and
|
||||
$kube_env.ContainsKey('EXEC_AUTH_PLUGIN_URL'))) {
|
||||
Log-Output -Fatal ("Missing one or more kube-env keys needed for " +
|
||||
"downloading auth plugin: $(Out-String $kube_env)")
|
||||
}
|
||||
MustDownload-File `
|
||||
-URLs ${kube_env}['EXEC_AUTH_PLUGIN_URL'] `
|
||||
-Hash ${kube_env}['EXEC_AUTH_PLUGIN_SHA1'] `
|
||||
-OutFile "${env:NODE_DIR}\gke-exec-auth-plugin.exe"
|
||||
MustDownload-File `
|
||||
-URLs ${kube_env}['EXEC_AUTH_PLUGIN_LICENSE_URL'] `
|
||||
-OutFile "${env:LICENSE_DIR}\LICENSE_gke-exec-auth-plugin.txt"
|
||||
}
|
||||
|
||||
# Downloads the Kubernetes binaries from kube-env's NODE_BINARY_TAR_URL and
|
||||
# puts them in a subdirectory of $env:K8S_DIR.
|
||||
#
|
||||
@ -477,43 +509,65 @@ function Write_PkiData {
|
||||
#
|
||||
# Required ${kube_env} keys:
|
||||
# CA_CERT
|
||||
# ${kube_env} keys that can be omitted for nodes that do not use an
|
||||
# authentication plugin:
|
||||
# KUBELET_CERT
|
||||
# KUBELET_KEY
|
||||
function Create-NodePki {
|
||||
Log-Output "Creating node pki files"
|
||||
Log-Output 'Creating node pki files'
|
||||
|
||||
if ($kube_env.ContainsKey('CA_CERT')) {
|
||||
$CA_CERT_BUNDLE = ${kube_env}['CA_CERT']
|
||||
$KUBELET_CERT = ${kube_env}['KUBELET_CERT']
|
||||
$KUBELET_KEY = ${kube_env}['KUBELET_KEY']
|
||||
Write_PkiData "${CA_CERT_BUNDLE}" ${env:CA_FILE_PATH}
|
||||
}
|
||||
else {
|
||||
Log-Output -Fatal 'CA_CERT not present in kube-env'
|
||||
}
|
||||
|
||||
Write_PkiData "${CA_CERT_BUNDLE}" ${env:CA_CERT_BUNDLE_PATH}
|
||||
# On nodes that use a plugin to support authentication, KUBELET_CERT and
|
||||
# KUBELET_KEY will not be present - TPM_BOOTSTRAP_CERT and TPM_BOOTSTRAP_KEY
|
||||
# should be set instead.
|
||||
if (Test-NodeUsesAuthPlugin ${kube_env}) {
|
||||
Log-Output ('Skipping KUBELET_CERT and KUBELET_KEY, plugin will be used ' +
|
||||
'for authentication')
|
||||
return
|
||||
}
|
||||
|
||||
if ($kube_env.ContainsKey('KUBELET_CERT')) {
|
||||
$KUBELET_CERT = ${kube_env}['KUBELET_CERT']
|
||||
Write_PkiData "${KUBELET_CERT}" ${env:KUBELET_CERT_PATH}
|
||||
}
|
||||
else {
|
||||
Log-Output -Fatal 'KUBELET_CERT not present in kube-env'
|
||||
}
|
||||
if ($kube_env.ContainsKey('KUBELET_KEY')) {
|
||||
$KUBELET_KEY = ${kube_env}['KUBELET_KEY']
|
||||
Write_PkiData "${KUBELET_KEY}" ${env:KUBELET_KEY_PATH}
|
||||
}
|
||||
else {
|
||||
Log-Output -Fatal 'KUBELET_KEY not present in kube-env'
|
||||
}
|
||||
|
||||
Get-ChildItem ${env:PKI_DIR}
|
||||
}
|
||||
|
||||
# Creates the kubelet kubeconfig at $env:BOOTSTRAP_KUBECONFIG.
|
||||
# Creates the bootstrap kubelet kubeconfig at $env:BOOTSTRAP_KUBECONFIG.
|
||||
# https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/
|
||||
#
|
||||
# Create-NodePki() must be called first.
|
||||
#
|
||||
# Required ${kube_env} keys:
|
||||
# KUBERNETES_MASTER_NAME: the apiserver IP address.
|
||||
function Create-KubeletKubeconfig {
|
||||
# The API server IP address comes from KUBERNETES_MASTER_NAME in kube-env, I
|
||||
# think. cluster/gce/gci/configure-helper.sh?l=2801
|
||||
$apiserverAddress = ${kube_env}['KUBERNETES_MASTER_NAME']
|
||||
|
||||
# TODO(pjh): set these using kube-env values.
|
||||
$createBootstrapConfig = $true
|
||||
$fetchBootstrapConfig = $false
|
||||
|
||||
if (${createBootstrapConfig}) {
|
||||
function Write_BootstrapKubeconfig {
|
||||
if (-not (ShouldWrite-File ${env:BOOTSTRAP_KUBECONFIG})) {
|
||||
return
|
||||
}
|
||||
New-Item -Force -ItemType file ${env:BOOTSTRAP_KUBECONFIG} | Out-Null
|
||||
|
||||
# TODO(mtaufen): is user "kubelet" correct? Other examples use e.g.
|
||||
# "system:node:$(hostname)".
|
||||
|
||||
$apiserverAddress = ${kube_env}['KUBERNETES_MASTER_NAME']
|
||||
New-Item -Force -ItemType file ${env:BOOTSTRAP_KUBECONFIG} | Out-Null
|
||||
Set-Content ${env:BOOTSTRAP_KUBECONFIG} `
|
||||
'apiVersion: v1
|
||||
kind: Config
|
||||
@ -526,7 +580,7 @@ clusters:
|
||||
- name: local
|
||||
cluster:
|
||||
server: https://APISERVER_ADDRESS
|
||||
certificate-authority: CA_CERT_BUNDLE_PATH
|
||||
certificate-authority: CA_FILE_PATH
|
||||
contexts:
|
||||
- context:
|
||||
cluster: local
|
||||
@ -536,20 +590,43 @@ current-context: service-account-context'.`
|
||||
replace('KUBELET_CERT_PATH', ${env:KUBELET_CERT_PATH}).`
|
||||
replace('KUBELET_KEY_PATH', ${env:KUBELET_KEY_PATH}).`
|
||||
replace('APISERVER_ADDRESS', ${apiserverAddress}).`
|
||||
replace('CA_CERT_BUNDLE_PATH', ${env:CA_CERT_BUNDLE_PATH})
|
||||
replace('CA_FILE_PATH', ${env:CA_FILE_PATH})
|
||||
Log-Output ("kubelet bootstrap kubeconfig:`n" +
|
||||
"$(Get-Content -Raw ${env:BOOTSTRAP_KUBECONFIG})")
|
||||
}
|
||||
elseif (${fetchBootstrapConfig}) {
|
||||
Log_NotImplemented `
|
||||
"fetching kubelet bootstrap-kubeconfig file from metadata"
|
||||
# get-metadata-value "instance/attributes/bootstrap-kubeconfig" >
|
||||
# /var/lib/kubelet/bootstrap-kubeconfig
|
||||
Log-Output ("kubelet bootstrap kubeconfig:`n" +
|
||||
"$(Get-Content -Raw ${env:BOOTSTRAP_KUBECONFIG})")
|
||||
|
||||
# Fetches the kubelet kubeconfig from the metadata server and writes it to
|
||||
# $env:KUBECONFIG.
|
||||
#
|
||||
# Create-NodePki() must be called first.
|
||||
function Write_KubeconfigFromMetadata {
|
||||
if (-not (ShouldWrite-File ${env:KUBECONFIG})) {
|
||||
return
|
||||
}
|
||||
else {
|
||||
Log_NotImplemented "fetching kubelet kubeconfig file from metadata"
|
||||
|
||||
$kubeconfig = Get-InstanceMetadataAttribute 'kubeconfig'
|
||||
if ($kubeconfig -eq $null) {
|
||||
Log-Output `
|
||||
"kubeconfig metadata key not found, can't write ${env:KUBECONFIG}" `
|
||||
-Fatal
|
||||
}
|
||||
Set-Content ${env:KUBECONFIG} $kubeconfig
|
||||
Log-Output ("kubelet kubeconfig from metadata (non-bootstrap):`n" +
|
||||
"$(Get-Content -Raw ${env:KUBECONFIG})")
|
||||
}
|
||||
|
||||
# Creates the kubelet kubeconfig at $env:KUBECONFIG for nodes that use an
|
||||
# authentication plugin, or at $env:BOOTSTRAP_KUBECONFIG for nodes that do not.
|
||||
#
|
||||
# Create-NodePki() must be called first.
|
||||
#
|
||||
# Required ${kube_env} keys:
|
||||
# KUBERNETES_MASTER_NAME: the apiserver IP address.
|
||||
function Create-KubeletKubeconfig {
|
||||
if (Test-NodeUsesAuthPlugin ${kube_env}) {
|
||||
Write_KubeconfigFromMetadata
|
||||
} else {
|
||||
Write_BootstrapKubeconfig
|
||||
}
|
||||
}
|
||||
|
||||
@ -1045,6 +1122,11 @@ function Start-WorkerServices {
|
||||
"--pod-infra-container-image=${INFRA_CONTAINER}"
|
||||
)
|
||||
$kubelet_args = ${default_kubelet_args} + ${kubelet_args}
|
||||
if (-not (Test-NodeUsesAuthPlugin ${kube_env})) {
|
||||
Log-Output 'Using bootstrap kubeconfig for authentication'
|
||||
$kubelet_args = (${kubelet_args} +
|
||||
"--bootstrap-kubeconfig=${env:BOOTSTRAP_KUBECONFIG}")
|
||||
}
|
||||
Log-Output "Final kubelet_args: ${kubelet_args}"
|
||||
|
||||
# Compute kube-proxy args
|
||||
|
||||
Loading…
Reference in New Issue
Block a user