mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 11:21:47 +00:00
Run kms-plugin in its own POD.
This commit is contained in:
immutablet
parent
e5964b3fcc
commit
4bc62be0e6
@ -409,7 +409,6 @@ function kube::release::package_kube_manifests_tarball() {
|
||||
cp "${src_dir}/cluster-autoscaler.manifest" "${dst_dir}/"
|
||||
cp "${src_dir}/etcd.manifest" "${dst_dir}"
|
||||
cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}"
|
||||
cp "${src_dir}/kms-plugin-container.manifest" "${dst_dir}"
|
||||
cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}"
|
||||
cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}"
|
||||
cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}"
|
||||
|
||||
@ -17,11 +17,13 @@ limitations under the License.
|
||||
package gci
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@ -53,87 +55,93 @@ readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
|
||||
readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
|
||||
readonly LOG_OWNER_USER=$(id -un)
|
||||
readonly LOG_OWNER_GROUP=$(id -gn)
|
||||
{{if .EncryptionProviderConfig}}
|
||||
ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
|
||||
{{end}}
|
||||
ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
|
||||
readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
|
||||
{{if .CloudKMSIntegration}}
|
||||
readonly CLOUD_KMS_INTEGRATION=true
|
||||
{{end}}
|
||||
`
|
||||
kubeAPIServerManifestFileName = "kube-apiserver.manifest"
|
||||
kmsPluginManifestFileName = "kms-plugin-container.manifest"
|
||||
kubeAPIServerStartFuncName = "start-kube-apiserver"
|
||||
|
||||
// Position of containers within a pod manifest
|
||||
kmsPluginContainerIndex = 0
|
||||
apiServerContainerIndexNoKMS = 0
|
||||
apiServerContainerIndexWithKMS = 1
|
||||
|
||||
// command": [
|
||||
// "/bin/sh", - Index 0
|
||||
// "-c", - Index 1
|
||||
// "exec /usr/local/bin/kube-apiserver " - Index 2
|
||||
execArgsIndex = 2
|
||||
|
||||
socketVolumeMountIndexKMSPlugin = 1
|
||||
socketVolumeMountIndexAPIServer = 0
|
||||
)
|
||||
|
||||
type kubeAPIServerEnv struct {
|
||||
KubeHome string
|
||||
EncryptionProviderConfig string
|
||||
EncryptionProviderConfigPath string
|
||||
ETCDKMSKeyID string
|
||||
EncryptionProviderConfig string
|
||||
CloudKMSIntegration bool
|
||||
}
|
||||
|
||||
type kubeAPIServerManifestTestCase struct {
|
||||
*ManifestTestCase
|
||||
apiServerContainer v1.Container
|
||||
kmsPluginContainer v1.Container
|
||||
}
|
||||
|
||||
func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
|
||||
return &kubeAPIServerManifestTestCase{
|
||||
ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
|
||||
ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
|
||||
func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv string) {
|
||||
c.mustInvokeFunc(kubeEnv, e)
|
||||
c.mustLoadPodFromManifest()
|
||||
|
||||
switch len(c.pod.Spec.Containers) {
|
||||
case 1:
|
||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
|
||||
case 2:
|
||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
|
||||
c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
|
||||
default:
|
||||
c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
|
||||
c.mustInvokeFunc(deployHelperEnv, e)
|
||||
c.mustLoadContainers()
|
||||
}
|
||||
|
||||
func getEncryptionProviderConfigFlag(path string) string {
|
||||
return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
|
||||
}
|
||||
|
||||
func TestEncryptionProviderFlag(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
var (
|
||||
// command": [
|
||||
// "/bin/sh", - Index 0
|
||||
// "-c", - Index 1
|
||||
// "exec /usr/local/bin/kube-apiserver " - Index 2
|
||||
execArgsIndex = 2
|
||||
encryptionConfigFlag = "--experimental-encryption-provider-config"
|
||||
)
|
||||
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
testCases := []struct {
|
||||
desc string
|
||||
encryptionProviderConfig string
|
||||
wantFlag bool
|
||||
}{
|
||||
{
|
||||
desc: "ENCRYPTION_PROVIDER_CONFIG is set",
|
||||
encryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")),
|
||||
wantFlag: true,
|
||||
},
|
||||
{
|
||||
desc: "ENCRYPTION_PROVIDER_CONFIG is not set",
|
||||
encryptionProviderConfig: "",
|
||||
wantFlag: false,
|
||||
},
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
||||
if !strings.Contains(execArgs, expectedFlag) {
|
||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
EncryptionProviderConfig: tc.encryptionProviderConfig,
|
||||
}
|
||||
|
||||
c.invokeTest(e, deployHelperEnv)
|
||||
|
||||
execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex]
|
||||
flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag)
|
||||
flag := fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath)
|
||||
|
||||
switch {
|
||||
case tc.wantFlag && !flagIsInArg:
|
||||
t.Fatalf("Got %q,\n want flags to contain %q", execArgs, flag)
|
||||
case !tc.wantFlag && flagIsInArg:
|
||||
t.Fatalf("Got %q,\n do not want flags to contain %q", execArgs, encryptionConfigFlag)
|
||||
case tc.wantFlag && flagIsInArg && !strings.Contains(execArgs, flag):
|
||||
t.Fatalf("Got flags: %q, want it to contain %q", execArgs, flag)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -144,8 +152,8 @@ func TestEncryptionProviderConfig(t *testing.T) {
|
||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
||||
EncryptionProviderConfigPath: p,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")),
|
||||
}
|
||||
|
||||
c.mustInvokeFunc(deployHelperEnv, e)
|
||||
@ -153,60 +161,91 @@ func TestEncryptionProviderConfig(t *testing.T) {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
|
||||
// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
|
||||
func TestKMSEncryptionProviderConfig(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
ETCDKMSKeyID: "FOO",
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
||||
if !strings.Contains(execArgs, expectedFlag) {
|
||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
||||
}
|
||||
|
||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||
}
|
||||
|
||||
d, err := ioutil.ReadFile(p)
|
||||
got, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to read encryption provider config %s", p)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(d), "name: grpc-kms-provider") {
|
||||
c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
|
||||
want := []byte("foo")
|
||||
if !bytes.Equal(got, want) {
|
||||
c.t.Fatalf("got encryptionConfig:\n%q\n, want encryptionConfig:\n%q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
var e = kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
ETCDKMSKeyID: "FOO",
|
||||
func TestKMSIntegration(t *testing.T) {
|
||||
var (
|
||||
socketPath = "/var/run/kmsplugin"
|
||||
dirOrCreate = v1.HostPathType(v1.HostPathDirectoryOrCreate)
|
||||
socketName = "kmssocket"
|
||||
)
|
||||
testCases := []struct {
|
||||
desc string
|
||||
cloudKMSIntegration bool
|
||||
wantVolume v1.Volume
|
||||
wantVolMount v1.VolumeMount
|
||||
}{
|
||||
{
|
||||
desc: "CLOUD_KMS_INTEGRATION is set",
|
||||
cloudKMSIntegration: true,
|
||||
wantVolume: v1.Volume{
|
||||
Name: socketName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: socketPath,
|
||||
Type: &dirOrCreate,
|
||||
},
|
||||
},
|
||||
},
|
||||
wantVolMount: v1.VolumeMount{
|
||||
Name: socketName,
|
||||
MountPath: socketPath,
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "CLOUD_KMS_INTEGRATION is not set",
|
||||
cloudKMSIntegration: false,
|
||||
},
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
|
||||
a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
|
||||
var e = kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")),
|
||||
CloudKMSIntegration: tc.cloudKMSIntegration,
|
||||
}
|
||||
|
||||
if k != a {
|
||||
t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
|
||||
k, a, k, a)
|
||||
c.invokeTest(e, deployHelperEnv)
|
||||
// By this point, we can be sure that kube-apiserver manifest is a valid POD.
|
||||
|
||||
var gotVolume v1.Volume
|
||||
for _, v := range c.pod.Spec.Volumes {
|
||||
if v.Name == socketName {
|
||||
gotVolume = v
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(gotVolume, tc.wantVolume) {
|
||||
t.Errorf("got volume %v, want %v", gotVolume, tc.wantVolume)
|
||||
}
|
||||
|
||||
var gotVolumeMount v1.VolumeMount
|
||||
for _, v := range c.pod.Spec.Containers[0].VolumeMounts {
|
||||
if v.Name == socketName {
|
||||
gotVolumeMount = v
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(gotVolumeMount, tc.wantVolMount) {
|
||||
t.Errorf("got volumeMount %v, want %v", gotVolumeMount, tc.wantVolMount)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -1771,29 +1771,11 @@ function start-kube-apiserver {
|
||||
container_env="\"env\":[{${container_env}}],"
|
||||
fi
|
||||
|
||||
if [[ -n "${ETCD_KMS_KEY_ID:-}" ]]; then
|
||||
ENCRYPTION_PROVIDER_CONFIG=$(cat << EOM | base64 | tr -d '\r\n'
|
||||
kind: EncryptionConfig
|
||||
apiVersion: v1
|
||||
resources:
|
||||
- resources:
|
||||
- secrets
|
||||
providers:
|
||||
- kms:
|
||||
name: grpc-kms-provider
|
||||
cachesize: 1000
|
||||
endpoint: unix:///var/run/kmsplugin/socket.sock
|
||||
EOM
|
||||
)
|
||||
fi
|
||||
local -r src_file="${src_dir}/kube-apiserver.manifest"
|
||||
|
||||
if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
|
||||
ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}"
|
||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
||||
params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
||||
fi
|
||||
# params is passed by reference, so no "$"
|
||||
setup-etcd-encryption "${src_file}" params
|
||||
|
||||
src_file="${src_dir}/kube-apiserver.manifest"
|
||||
# Evaluate variables.
|
||||
local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
|
||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||
@ -1822,67 +1804,116 @@ EOM
|
||||
sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}"
|
||||
sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}"
|
||||
|
||||
if [[ -z "${ETCD_KMS_KEY_ID:-}" ]]; then
|
||||
# Removing KMS related placeholders.
|
||||
sed -i -e " {
|
||||
s@{{kms_plugin_container}}@@
|
||||
cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
|
||||
}
|
||||
|
||||
s@{{kms_socket_mount}}@@
|
||||
# Sets-up etcd encryption.
|
||||
# Configuration of etcd level encryption consists of the following steps:
|
||||
# 1. Writing encryption provider config to disk
|
||||
# 2. Adding experimental-encryption-provider-config flag to kube-apiserver
|
||||
# 3. Add kms-socket-vol and kms-socket-vol-mnt to enable communication with kms-plugin (if requested)
|
||||
#
|
||||
# Expects parameters:
|
||||
# $1 - path to kube-apiserver template
|
||||
# $2 - kube-apiserver startup flags (must be passed by reference)
|
||||
#
|
||||
# Assumes vars (supplied via kube-env):
|
||||
# ENCRYPTION_PROVIDER_CONFIG
|
||||
# CLOUD_KMS_INTEGRATION
|
||||
# ENCRYPTION_PROVIDER_CONFIG_PATH (will default to /etc/srv/kubernetes/encryption-provider-config.yml)
|
||||
function setup-etcd-encryption {
|
||||
local kube_apiserver_template_path
|
||||
local -n kube_api_server_params
|
||||
local default_encryption_provider_config_vol
|
||||
local default_encryption_provider_config_vol_mnt
|
||||
local encryption_provider_config_vol_mnt
|
||||
local encryption_provider_config_vol
|
||||
local default_kms_socket_dir
|
||||
local default_kms_socket_vol_mnt
|
||||
local default_kms_socket_vol
|
||||
local kms_socket_vol_mnt
|
||||
local kms_socket_vol
|
||||
local encryption_provider_config_path
|
||||
|
||||
kube_apiserver_template_path="$1"
|
||||
if [[ -z "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
|
||||
sed -i -e " {
|
||||
s@{{encryption_provider_mount}}@@
|
||||
|
||||
s@{{kms_socket_volume}}@@
|
||||
s@{{encryption_provider_volume}}@@
|
||||
} " "${src_file}"
|
||||
else
|
||||
local kms_plugin_src_file="${src_dir}/kms-plugin-container.manifest"
|
||||
|
||||
if [[ ! -f "${kms_plugin_src_file}" ]]; then
|
||||
echo "Error: KMS Integration was requested, but "${kms_plugin_src_file}" is missing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then
|
||||
echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# TODO: Validate that the encryption config is for KMS.
|
||||
|
||||
local kms_socket_dir="/var/run/kmsplugin"
|
||||
|
||||
# kms_socket_mnt is used by both kms_plugin and kube-apiserver - this is how these containers talk.
|
||||
local kms_socket_mnt="{ \"name\": \"kmssocket\", \"mountPath\": \"${kms_socket_dir}\", \"readOnly\": false}"
|
||||
|
||||
local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
|
||||
local kms_path_to_socket="${kms_socket_dir}/socket.sock"
|
||||
|
||||
local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}"
|
||||
local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}"
|
||||
|
||||
# TODO these are used in other places, convert to global.
|
||||
local gce_conf_path="/etc/gce.conf"
|
||||
local cloud_config_mount="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}"
|
||||
|
||||
local kms_plugin_container=$(echo $(sed " {
|
||||
s@{{kms_key_uri}}@${ETCD_KMS_KEY_ID}@
|
||||
s@{{gce_conf_path}}@${gce_conf_path}@
|
||||
s@{{kms_path_to_socket}}@${kms_path_to_socket}@
|
||||
s@{{kms_socket_mount}}@${kms_socket_mnt}@
|
||||
s@{{cloud_config_mount}}@${cloud_config_mount}@
|
||||
} " "${kms_plugin_src_file}") | tr "\n" "\\n")
|
||||
|
||||
sed -i -e " {
|
||||
s@{{kms_plugin_container}}@${kms_plugin_container},@
|
||||
|
||||
s@{{kms_socket_mount}}@${kms_socket_mnt},@
|
||||
s@{{encryption_provider_mount}}@${encryption_provider_mnt},@
|
||||
|
||||
s@{{kms_socket_volume}}@${kms_socket_vol},@
|
||||
s@{{encryption_provider_volume}}@${encryption_provider_vol},@
|
||||
} " "${src_file}"
|
||||
s@{{kms_socket_mount}}@@
|
||||
s@{{kms_socket_volume}}@@
|
||||
} " "${kube_apiserver_template_path}"
|
||||
return
|
||||
fi
|
||||
|
||||
cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
|
||||
kube_api_server_params="$2"
|
||||
encryption_provider_config_path=${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}
|
||||
|
||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
|
||||
kube_api_server_params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
|
||||
|
||||
default_encryption_provider_config_vol=$(echo "{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" | base64 | tr -d '\r\n')
|
||||
default_encryption_provider_config_vol_mnt=$(echo "{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" | base64 | tr -d '\r\n')
|
||||
|
||||
encryption_provider_config_vol_mnt=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL_MNT:-"${default_encryption_provider_config_vol_mnt}"}" | base64 --decode)
|
||||
encryption_provider_config_vol=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL:-"${default_encryption_provider_config_vol}"}" | base64 --decode)
|
||||
sed -i -e " {
|
||||
s@{{encryption_provider_mount}}@${encryption_provider_config_vol_mnt},@
|
||||
s@{{encryption_provider_volume}}@${encryption_provider_config_vol},@
|
||||
} " "${kube_apiserver_template_path}"
|
||||
|
||||
if [[ -n "${CLOUD_KMS_INTEGRATION:-}" ]]; then
|
||||
default_kms_socket_dir="/var/run/kmsplugin"
|
||||
default_kms_socket_vol_mnt=$(echo "{ \"name\": \"kmssocket\", \"mountPath\": \"${default_kms_socket_dir}\", \"readOnly\": false}" | base64 | tr -d '\r\n')
|
||||
default_kms_socket_vol=$(echo "{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${default_kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" | base64 | tr -d '\r\n')
|
||||
|
||||
kms_socket_vol_mnt=$(echo "${KMS_PLUGIN_SOCKET_VOL_MNT:-"${default_kms_socket_vol_mnt}"}" | base64 --decode)
|
||||
kms_socket_vol=$(echo "${KMS_PLUGIN_SOCKET_VOL:-"${default_kms_socket_vol}"}" | base64 --decode)
|
||||
sed -i -e " {
|
||||
s@{{kms_socket_mount}}@${kms_socket_vol_mnt},@
|
||||
s@{{kms_socket_volume}}@${kms_socket_vol},@
|
||||
} " "${kube_apiserver_template_path}"
|
||||
else
|
||||
sed -i -e " {
|
||||
s@{{kms_socket_mount}}@@
|
||||
s@{{kms_socket_volume}}@@
|
||||
} " "${kube_apiserver_template_path}"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
# Applies encryption provider config.
|
||||
# This function may be triggered in two scenarios:
|
||||
# 1. Decryption of etcd
|
||||
# 2. Encryption of etcd is added after the cluster is deployed
|
||||
# Both cases require that the existing secrets in etcd be re-proceeded.
|
||||
#
|
||||
# Assumes vars (supplied via kube-env):
|
||||
# ENCRYPTION_PROVIDER_CONFIG_FORCE
|
||||
function apply-encryption-config() {
|
||||
if [[ "${ENCRYPTION_PROVIDER_CONFIG_FORCE:-false}" == "false" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# need kube-apiserver to be ready
|
||||
until kubectl get secret; do
|
||||
sleep ${ENCRYPTION_PROVIDER_CONFIG_FORCE_DELAY:-5}
|
||||
done
|
||||
|
||||
retries=${ENCRYPTION_PROVIDER_CONFIG_FORCE_RETRIES:-5}
|
||||
# The command below may fail when a conflict is detected during an update on a secret (something
|
||||
# else updated the secret in the middle of our update).
|
||||
# TODO: Retry only on errors caused by a conflict.
|
||||
until (( retries == 0 )); do
|
||||
# forces all secrets to be re-written to etcd, and in the process either encrypting or decrypting them
|
||||
# https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/
|
||||
if kubectl get secrets --all-namespaces -o json | kubectl replace -f -; then
|
||||
break
|
||||
fi
|
||||
|
||||
(( retries-- ))
|
||||
sleep "${ENCRYPTION_PROVIDER_CONFIG_FORCE_RETRY_SLEEP:-3}"
|
||||
done
|
||||
}
|
||||
|
||||
# Starts kubernetes controller manager.
|
||||
@ -2778,6 +2809,7 @@ function main() {
|
||||
start-kube-addons
|
||||
start-cluster-autoscaler
|
||||
start-lb-controller
|
||||
apply-encryption-config &
|
||||
else
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
start-kube-proxy
|
||||
|
||||
@ -143,7 +143,7 @@ func (c *ManifestTestCase) mustLoadPodFromManifest() {
|
||||
}
|
||||
|
||||
if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
|
||||
c.t.Fatalf("Failed to decode manifest: %v", err)
|
||||
c.t.Fatalf("Failed to decode manifest:\n%s\nerror: %v", json, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -18,7 +18,6 @@ filegroup(
|
||||
"etcd.manifest",
|
||||
"etcd-empty-dir-cleanup.yaml",
|
||||
"glbc.manifest",
|
||||
"kms-plugin-container.manifest",
|
||||
"kube-addon-manager.yaml",
|
||||
"kube-apiserver.manifest",
|
||||
"kube-controller-manager.manifest",
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
{
|
||||
"name": "kms-plugin",
|
||||
"image": "gcr.io/google-containers/k8s-cloud-kms-plugin:v0.1.1",
|
||||
"command": ["/k8s-cloud-kms-plugin", "--key-uri={{kms_key_uri}}", "--path-to-unix-socket={{kms_path_to_socket}}", "--gce-config={{gce_conf_path}}", "--logtostderr", "2>\&1"],
|
||||
"livenessProbe": { "httpGet": {"host": "127.0.0.1", "port": 8081, "path": "/healthz"}, "initialDelaySeconds": 3, "timeoutSeconds": 3},
|
||||
"ports":[{ "name": "healthz", "containerPort": 8081, "hostPort": 8081}, { "name": "metrics", "containerPort": 8082, "hostPort": 8082}],
|
||||
"volumeMounts": [{{cloud_config_mount}}, {{kms_socket_mount}}]
|
||||
}
|
||||
@ -16,7 +16,6 @@
|
||||
"spec":{
|
||||
"hostNetwork": true,
|
||||
"containers":[
|
||||
{{kms_plugin_container}}
|
||||
{
|
||||
"name": "kube-apiserver",
|
||||
"image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user