mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Run kms-plugin in its own POD.
This commit is contained in:
parent
e5964b3fcc
commit
4bc62be0e6
@ -409,7 +409,6 @@ function kube::release::package_kube_manifests_tarball() {
|
|||||||
cp "${src_dir}/cluster-autoscaler.manifest" "${dst_dir}/"
|
cp "${src_dir}/cluster-autoscaler.manifest" "${dst_dir}/"
|
||||||
cp "${src_dir}/etcd.manifest" "${dst_dir}"
|
cp "${src_dir}/etcd.manifest" "${dst_dir}"
|
||||||
cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}"
|
cp "${src_dir}/kube-scheduler.manifest" "${dst_dir}"
|
||||||
cp "${src_dir}/kms-plugin-container.manifest" "${dst_dir}"
|
|
||||||
cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}"
|
cp "${src_dir}/kube-apiserver.manifest" "${dst_dir}"
|
||||||
cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}"
|
cp "${src_dir}/abac-authz-policy.jsonl" "${dst_dir}"
|
||||||
cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}"
|
cp "${src_dir}/kube-controller-manager.manifest" "${dst_dir}"
|
||||||
|
@ -17,11 +17,13 @@ limitations under the License.
|
|||||||
package gci
|
package gci
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@ -53,87 +55,93 @@ readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
|
|||||||
readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
|
readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
|
||||||
readonly LOG_OWNER_USER=$(id -un)
|
readonly LOG_OWNER_USER=$(id -un)
|
||||||
readonly LOG_OWNER_GROUP=$(id -gn)
|
readonly LOG_OWNER_GROUP=$(id -gn)
|
||||||
|
{{if .EncryptionProviderConfig}}
|
||||||
ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
|
ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
|
||||||
|
{{end}}
|
||||||
ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
|
ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
|
||||||
readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
|
{{if .CloudKMSIntegration}}
|
||||||
|
readonly CLOUD_KMS_INTEGRATION=true
|
||||||
|
{{end}}
|
||||||
`
|
`
|
||||||
kubeAPIServerManifestFileName = "kube-apiserver.manifest"
|
kubeAPIServerManifestFileName = "kube-apiserver.manifest"
|
||||||
kmsPluginManifestFileName = "kms-plugin-container.manifest"
|
|
||||||
kubeAPIServerStartFuncName = "start-kube-apiserver"
|
kubeAPIServerStartFuncName = "start-kube-apiserver"
|
||||||
|
|
||||||
// Position of containers within a pod manifest
|
|
||||||
kmsPluginContainerIndex = 0
|
|
||||||
apiServerContainerIndexNoKMS = 0
|
|
||||||
apiServerContainerIndexWithKMS = 1
|
|
||||||
|
|
||||||
// command": [
|
|
||||||
// "/bin/sh", - Index 0
|
|
||||||
// "-c", - Index 1
|
|
||||||
// "exec /usr/local/bin/kube-apiserver " - Index 2
|
|
||||||
execArgsIndex = 2
|
|
||||||
|
|
||||||
socketVolumeMountIndexKMSPlugin = 1
|
|
||||||
socketVolumeMountIndexAPIServer = 0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type kubeAPIServerEnv struct {
|
type kubeAPIServerEnv struct {
|
||||||
KubeHome string
|
KubeHome string
|
||||||
EncryptionProviderConfig string
|
|
||||||
EncryptionProviderConfigPath string
|
EncryptionProviderConfigPath string
|
||||||
ETCDKMSKeyID string
|
EncryptionProviderConfig string
|
||||||
|
CloudKMSIntegration bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type kubeAPIServerManifestTestCase struct {
|
type kubeAPIServerManifestTestCase struct {
|
||||||
*ManifestTestCase
|
*ManifestTestCase
|
||||||
apiServerContainer v1.Container
|
|
||||||
kmsPluginContainer v1.Container
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
|
func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
|
||||||
return &kubeAPIServerManifestTestCase{
|
return &kubeAPIServerManifestTestCase{
|
||||||
ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
|
ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, nil),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
|
func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv string) {
|
||||||
|
c.mustInvokeFunc(kubeEnv, e)
|
||||||
c.mustLoadPodFromManifest()
|
c.mustLoadPodFromManifest()
|
||||||
|
|
||||||
switch len(c.pod.Spec.Containers) {
|
|
||||||
case 1:
|
|
||||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
|
|
||||||
case 2:
|
|
||||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
|
|
||||||
c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
|
|
||||||
default:
|
|
||||||
c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
|
|
||||||
c.mustInvokeFunc(deployHelperEnv, e)
|
|
||||||
c.mustLoadContainers()
|
|
||||||
}
|
|
||||||
|
|
||||||
func getEncryptionProviderConfigFlag(path string) string {
|
|
||||||
return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEncryptionProviderFlag(t *testing.T) {
|
func TestEncryptionProviderFlag(t *testing.T) {
|
||||||
c := newKubeAPIServerManifestTestCase(t)
|
var (
|
||||||
defer c.tearDown()
|
// command": [
|
||||||
|
// "/bin/sh", - Index 0
|
||||||
|
// "-c", - Index 1
|
||||||
|
// "exec /usr/local/bin/kube-apiserver " - Index 2
|
||||||
|
execArgsIndex = 2
|
||||||
|
encryptionConfigFlag = "--experimental-encryption-provider-config"
|
||||||
|
)
|
||||||
|
|
||||||
e := kubeAPIServerEnv{
|
testCases := []struct {
|
||||||
KubeHome: c.kubeHome,
|
desc string
|
||||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
encryptionProviderConfig string
|
||||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
wantFlag bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
desc: "ENCRYPTION_PROVIDER_CONFIG is set",
|
||||||
|
encryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")),
|
||||||
|
wantFlag: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "ENCRYPTION_PROVIDER_CONFIG is not set",
|
||||||
|
encryptionProviderConfig: "",
|
||||||
|
wantFlag: false,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
c.invokeTest(e)
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
|
c := newKubeAPIServerManifestTestCase(t)
|
||||||
|
defer c.tearDown()
|
||||||
|
|
||||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
e := kubeAPIServerEnv{
|
||||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
KubeHome: c.kubeHome,
|
||||||
if !strings.Contains(execArgs, expectedFlag) {
|
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
EncryptionProviderConfig: tc.encryptionProviderConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
c.invokeTest(e, deployHelperEnv)
|
||||||
|
|
||||||
|
execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex]
|
||||||
|
flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag)
|
||||||
|
flag := fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case tc.wantFlag && !flagIsInArg:
|
||||||
|
t.Fatalf("Got %q,\n want flags to contain %q", execArgs, flag)
|
||||||
|
case !tc.wantFlag && flagIsInArg:
|
||||||
|
t.Fatalf("Got %q,\n do not want flags to contain %q", execArgs, encryptionConfigFlag)
|
||||||
|
case tc.wantFlag && flagIsInArg && !strings.Contains(execArgs, flag):
|
||||||
|
t.Fatalf("Got flags: %q, want it to contain %q", execArgs, flag)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -144,8 +152,8 @@ func TestEncryptionProviderConfig(t *testing.T) {
|
|||||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||||
e := kubeAPIServerEnv{
|
e := kubeAPIServerEnv{
|
||||||
KubeHome: c.kubeHome,
|
KubeHome: c.kubeHome,
|
||||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
|
||||||
EncryptionProviderConfigPath: p,
|
EncryptionProviderConfigPath: p,
|
||||||
|
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")),
|
||||||
}
|
}
|
||||||
|
|
||||||
c.mustInvokeFunc(deployHelperEnv, e)
|
c.mustInvokeFunc(deployHelperEnv, e)
|
||||||
@ -153,60 +161,91 @@ func TestEncryptionProviderConfig(t *testing.T) {
|
|||||||
if _, err := os.Stat(p); err != nil {
|
if _, err := os.Stat(p); err != nil {
|
||||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
|
got, err := ioutil.ReadFile(p)
|
||||||
// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
|
|
||||||
func TestKMSEncryptionProviderConfig(t *testing.T) {
|
|
||||||
c := newKubeAPIServerManifestTestCase(t)
|
|
||||||
defer c.tearDown()
|
|
||||||
|
|
||||||
e := kubeAPIServerEnv{
|
|
||||||
KubeHome: c.kubeHome,
|
|
||||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
|
||||||
ETCDKMSKeyID: "FOO",
|
|
||||||
}
|
|
||||||
|
|
||||||
c.invokeTest(e)
|
|
||||||
|
|
||||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
|
||||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
|
||||||
if !strings.Contains(execArgs, expectedFlag) {
|
|
||||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
|
||||||
}
|
|
||||||
|
|
||||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
|
||||||
if _, err := os.Stat(p); err != nil {
|
|
||||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
d, err := ioutil.ReadFile(p)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.t.Fatalf("Failed to read encryption provider config %s", p)
|
c.t.Fatalf("Failed to read encryption provider config %s", p)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(string(d), "name: grpc-kms-provider") {
|
want := []byte("foo")
|
||||||
c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
|
if !bytes.Equal(got, want) {
|
||||||
|
c.t.Fatalf("got encryptionConfig:\n%q\n, want encryptionConfig:\n%q", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
|
func TestKMSIntegration(t *testing.T) {
|
||||||
c := newKubeAPIServerManifestTestCase(t)
|
var (
|
||||||
defer c.tearDown()
|
socketPath = "/var/run/kmsplugin"
|
||||||
|
dirOrCreate = v1.HostPathType(v1.HostPathDirectoryOrCreate)
|
||||||
var e = kubeAPIServerEnv{
|
socketName = "kmssocket"
|
||||||
KubeHome: c.kubeHome,
|
)
|
||||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
testCases := []struct {
|
||||||
ETCDKMSKeyID: "FOO",
|
desc string
|
||||||
|
cloudKMSIntegration bool
|
||||||
|
wantVolume v1.Volume
|
||||||
|
wantVolMount v1.VolumeMount
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
desc: "CLOUD_KMS_INTEGRATION is set",
|
||||||
|
cloudKMSIntegration: true,
|
||||||
|
wantVolume: v1.Volume{
|
||||||
|
Name: socketName,
|
||||||
|
VolumeSource: v1.VolumeSource{
|
||||||
|
HostPath: &v1.HostPathVolumeSource{
|
||||||
|
Path: socketPath,
|
||||||
|
Type: &dirOrCreate,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
wantVolMount: v1.VolumeMount{
|
||||||
|
Name: socketName,
|
||||||
|
MountPath: socketPath,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "CLOUD_KMS_INTEGRATION is not set",
|
||||||
|
cloudKMSIntegration: false,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
c.invokeTest(e)
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
|
c := newKubeAPIServerManifestTestCase(t)
|
||||||
|
defer c.tearDown()
|
||||||
|
|
||||||
k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
|
var e = kubeAPIServerEnv{
|
||||||
a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
|
KubeHome: c.kubeHome,
|
||||||
|
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||||
|
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("foo")),
|
||||||
|
CloudKMSIntegration: tc.cloudKMSIntegration,
|
||||||
|
}
|
||||||
|
|
||||||
if k != a {
|
c.invokeTest(e, deployHelperEnv)
|
||||||
t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
|
// By this point, we can be sure that kube-apiserver manifest is a valid POD.
|
||||||
k, a, k, a)
|
|
||||||
|
var gotVolume v1.Volume
|
||||||
|
for _, v := range c.pod.Spec.Volumes {
|
||||||
|
if v.Name == socketName {
|
||||||
|
gotVolume = v
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(gotVolume, tc.wantVolume) {
|
||||||
|
t.Errorf("got volume %v, want %v", gotVolume, tc.wantVolume)
|
||||||
|
}
|
||||||
|
|
||||||
|
var gotVolumeMount v1.VolumeMount
|
||||||
|
for _, v := range c.pod.Spec.Containers[0].VolumeMounts {
|
||||||
|
if v.Name == socketName {
|
||||||
|
gotVolumeMount = v
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(gotVolumeMount, tc.wantVolMount) {
|
||||||
|
t.Errorf("got volumeMount %v, want %v", gotVolumeMount, tc.wantVolMount)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1771,29 +1771,11 @@ function start-kube-apiserver {
|
|||||||
container_env="\"env\":[{${container_env}}],"
|
container_env="\"env\":[{${container_env}}],"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -n "${ETCD_KMS_KEY_ID:-}" ]]; then
|
local -r src_file="${src_dir}/kube-apiserver.manifest"
|
||||||
ENCRYPTION_PROVIDER_CONFIG=$(cat << EOM | base64 | tr -d '\r\n'
|
|
||||||
kind: EncryptionConfig
|
|
||||||
apiVersion: v1
|
|
||||||
resources:
|
|
||||||
- resources:
|
|
||||||
- secrets
|
|
||||||
providers:
|
|
||||||
- kms:
|
|
||||||
name: grpc-kms-provider
|
|
||||||
cachesize: 1000
|
|
||||||
endpoint: unix:///var/run/kmsplugin/socket.sock
|
|
||||||
EOM
|
|
||||||
)
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
|
# params is passed by reference, so no "$"
|
||||||
ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}"
|
setup-etcd-encryption "${src_file}" params
|
||||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
|
||||||
params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
src_file="${src_dir}/kube-apiserver.manifest"
|
|
||||||
# Evaluate variables.
|
# Evaluate variables.
|
||||||
local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
|
local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
|
||||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||||
@ -1822,67 +1804,116 @@ EOM
|
|||||||
sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}"
|
sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}"
|
||||||
sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}"
|
sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}"
|
||||||
|
|
||||||
if [[ -z "${ETCD_KMS_KEY_ID:-}" ]]; then
|
cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
|
||||||
# Removing KMS related placeholders.
|
}
|
||||||
sed -i -e " {
|
|
||||||
s@{{kms_plugin_container}}@@
|
|
||||||
|
|
||||||
s@{{kms_socket_mount}}@@
|
# Sets-up etcd encryption.
|
||||||
|
# Configuration of etcd level encryption consists of the following steps:
|
||||||
|
# 1. Writing encryption provider config to disk
|
||||||
|
# 2. Adding experimental-encryption-provider-config flag to kube-apiserver
|
||||||
|
# 3. Add kms-socket-vol and kms-socket-vol-mnt to enable communication with kms-plugin (if requested)
|
||||||
|
#
|
||||||
|
# Expects parameters:
|
||||||
|
# $1 - path to kube-apiserver template
|
||||||
|
# $2 - kube-apiserver startup flags (must be passed by reference)
|
||||||
|
#
|
||||||
|
# Assumes vars (supplied via kube-env):
|
||||||
|
# ENCRYPTION_PROVIDER_CONFIG
|
||||||
|
# CLOUD_KMS_INTEGRATION
|
||||||
|
# ENCRYPTION_PROVIDER_CONFIG_PATH (will default to /etc/srv/kubernetes/encryption-provider-config.yml)
|
||||||
|
function setup-etcd-encryption {
|
||||||
|
local kube_apiserver_template_path
|
||||||
|
local -n kube_api_server_params
|
||||||
|
local default_encryption_provider_config_vol
|
||||||
|
local default_encryption_provider_config_vol_mnt
|
||||||
|
local encryption_provider_config_vol_mnt
|
||||||
|
local encryption_provider_config_vol
|
||||||
|
local default_kms_socket_dir
|
||||||
|
local default_kms_socket_vol_mnt
|
||||||
|
local default_kms_socket_vol
|
||||||
|
local kms_socket_vol_mnt
|
||||||
|
local kms_socket_vol
|
||||||
|
local encryption_provider_config_path
|
||||||
|
|
||||||
|
kube_apiserver_template_path="$1"
|
||||||
|
if [[ -z "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
|
||||||
|
sed -i -e " {
|
||||||
s@{{encryption_provider_mount}}@@
|
s@{{encryption_provider_mount}}@@
|
||||||
|
|
||||||
s@{{kms_socket_volume}}@@
|
|
||||||
s@{{encryption_provider_volume}}@@
|
s@{{encryption_provider_volume}}@@
|
||||||
} " "${src_file}"
|
s@{{kms_socket_mount}}@@
|
||||||
else
|
s@{{kms_socket_volume}}@@
|
||||||
local kms_plugin_src_file="${src_dir}/kms-plugin-container.manifest"
|
} " "${kube_apiserver_template_path}"
|
||||||
|
return
|
||||||
if [[ ! -f "${kms_plugin_src_file}" ]]; then
|
|
||||||
echo "Error: KMS Integration was requested, but "${kms_plugin_src_file}" is missing."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then
|
|
||||||
echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# TODO: Validate that the encryption config is for KMS.
|
|
||||||
|
|
||||||
local kms_socket_dir="/var/run/kmsplugin"
|
|
||||||
|
|
||||||
# kms_socket_mnt is used by both kms_plugin and kube-apiserver - this is how these containers talk.
|
|
||||||
local kms_socket_mnt="{ \"name\": \"kmssocket\", \"mountPath\": \"${kms_socket_dir}\", \"readOnly\": false}"
|
|
||||||
|
|
||||||
local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
|
|
||||||
local kms_path_to_socket="${kms_socket_dir}/socket.sock"
|
|
||||||
|
|
||||||
local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}"
|
|
||||||
local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}"
|
|
||||||
|
|
||||||
# TODO these are used in other places, convert to global.
|
|
||||||
local gce_conf_path="/etc/gce.conf"
|
|
||||||
local cloud_config_mount="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}"
|
|
||||||
|
|
||||||
local kms_plugin_container=$(echo $(sed " {
|
|
||||||
s@{{kms_key_uri}}@${ETCD_KMS_KEY_ID}@
|
|
||||||
s@{{gce_conf_path}}@${gce_conf_path}@
|
|
||||||
s@{{kms_path_to_socket}}@${kms_path_to_socket}@
|
|
||||||
s@{{kms_socket_mount}}@${kms_socket_mnt}@
|
|
||||||
s@{{cloud_config_mount}}@${cloud_config_mount}@
|
|
||||||
} " "${kms_plugin_src_file}") | tr "\n" "\\n")
|
|
||||||
|
|
||||||
sed -i -e " {
|
|
||||||
s@{{kms_plugin_container}}@${kms_plugin_container},@
|
|
||||||
|
|
||||||
s@{{kms_socket_mount}}@${kms_socket_mnt},@
|
|
||||||
s@{{encryption_provider_mount}}@${encryption_provider_mnt},@
|
|
||||||
|
|
||||||
s@{{kms_socket_volume}}@${kms_socket_vol},@
|
|
||||||
s@{{encryption_provider_volume}}@${encryption_provider_vol},@
|
|
||||||
} " "${src_file}"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
|
kube_api_server_params="$2"
|
||||||
|
encryption_provider_config_path=${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}
|
||||||
|
|
||||||
|
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
|
||||||
|
kube_api_server_params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
|
||||||
|
|
||||||
|
default_encryption_provider_config_vol=$(echo "{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" | base64 | tr -d '\r\n')
|
||||||
|
default_encryption_provider_config_vol_mnt=$(echo "{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" | base64 | tr -d '\r\n')
|
||||||
|
|
||||||
|
encryption_provider_config_vol_mnt=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL_MNT:-"${default_encryption_provider_config_vol_mnt}"}" | base64 --decode)
|
||||||
|
encryption_provider_config_vol=$(echo "${ENCRYPTION_PROVIDER_CONFIG_VOL:-"${default_encryption_provider_config_vol}"}" | base64 --decode)
|
||||||
|
sed -i -e " {
|
||||||
|
s@{{encryption_provider_mount}}@${encryption_provider_config_vol_mnt},@
|
||||||
|
s@{{encryption_provider_volume}}@${encryption_provider_config_vol},@
|
||||||
|
} " "${kube_apiserver_template_path}"
|
||||||
|
|
||||||
|
if [[ -n "${CLOUD_KMS_INTEGRATION:-}" ]]; then
|
||||||
|
default_kms_socket_dir="/var/run/kmsplugin"
|
||||||
|
default_kms_socket_vol_mnt=$(echo "{ \"name\": \"kmssocket\", \"mountPath\": \"${default_kms_socket_dir}\", \"readOnly\": false}" | base64 | tr -d '\r\n')
|
||||||
|
default_kms_socket_vol=$(echo "{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${default_kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" | base64 | tr -d '\r\n')
|
||||||
|
|
||||||
|
kms_socket_vol_mnt=$(echo "${KMS_PLUGIN_SOCKET_VOL_MNT:-"${default_kms_socket_vol_mnt}"}" | base64 --decode)
|
||||||
|
kms_socket_vol=$(echo "${KMS_PLUGIN_SOCKET_VOL:-"${default_kms_socket_vol}"}" | base64 --decode)
|
||||||
|
sed -i -e " {
|
||||||
|
s@{{kms_socket_mount}}@${kms_socket_vol_mnt},@
|
||||||
|
s@{{kms_socket_volume}}@${kms_socket_vol},@
|
||||||
|
} " "${kube_apiserver_template_path}"
|
||||||
|
else
|
||||||
|
sed -i -e " {
|
||||||
|
s@{{kms_socket_mount}}@@
|
||||||
|
s@{{kms_socket_volume}}@@
|
||||||
|
} " "${kube_apiserver_template_path}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Applies encryption provider config.
|
||||||
|
# This function may be triggered in two scenarios:
|
||||||
|
# 1. Decryption of etcd
|
||||||
|
# 2. Encryption of etcd is added after the cluster is deployed
|
||||||
|
# Both cases require that the existing secrets in etcd be re-proceeded.
|
||||||
|
#
|
||||||
|
# Assumes vars (supplied via kube-env):
|
||||||
|
# ENCRYPTION_PROVIDER_CONFIG_FORCE
|
||||||
|
function apply-encryption-config() {
|
||||||
|
if [[ "${ENCRYPTION_PROVIDER_CONFIG_FORCE:-false}" == "false" ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
# need kube-apiserver to be ready
|
||||||
|
until kubectl get secret; do
|
||||||
|
sleep ${ENCRYPTION_PROVIDER_CONFIG_FORCE_DELAY:-5}
|
||||||
|
done
|
||||||
|
|
||||||
|
retries=${ENCRYPTION_PROVIDER_CONFIG_FORCE_RETRIES:-5}
|
||||||
|
# The command below may fail when a conflict is detected during an update on a secret (something
|
||||||
|
# else updated the secret in the middle of our update).
|
||||||
|
# TODO: Retry only on errors caused by a conflict.
|
||||||
|
until (( retries == 0 )); do
|
||||||
|
# forces all secrets to be re-written to etcd, and in the process either encrypting or decrypting them
|
||||||
|
# https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/
|
||||||
|
if kubectl get secrets --all-namespaces -o json | kubectl replace -f -; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
(( retries-- ))
|
||||||
|
sleep "${ENCRYPTION_PROVIDER_CONFIG_FORCE_RETRY_SLEEP:-3}"
|
||||||
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
# Starts kubernetes controller manager.
|
# Starts kubernetes controller manager.
|
||||||
@ -2778,6 +2809,7 @@ function main() {
|
|||||||
start-kube-addons
|
start-kube-addons
|
||||||
start-cluster-autoscaler
|
start-cluster-autoscaler
|
||||||
start-lb-controller
|
start-lb-controller
|
||||||
|
apply-encryption-config &
|
||||||
else
|
else
|
||||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||||
start-kube-proxy
|
start-kube-proxy
|
||||||
|
@ -143,7 +143,7 @@ func (c *ManifestTestCase) mustLoadPodFromManifest() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
|
if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
|
||||||
c.t.Fatalf("Failed to decode manifest: %v", err)
|
c.t.Fatalf("Failed to decode manifest:\n%s\nerror: %v", json, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -18,7 +18,6 @@ filegroup(
|
|||||||
"etcd.manifest",
|
"etcd.manifest",
|
||||||
"etcd-empty-dir-cleanup.yaml",
|
"etcd-empty-dir-cleanup.yaml",
|
||||||
"glbc.manifest",
|
"glbc.manifest",
|
||||||
"kms-plugin-container.manifest",
|
|
||||||
"kube-addon-manager.yaml",
|
"kube-addon-manager.yaml",
|
||||||
"kube-apiserver.manifest",
|
"kube-apiserver.manifest",
|
||||||
"kube-controller-manager.manifest",
|
"kube-controller-manager.manifest",
|
||||||
|
@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "kms-plugin",
|
|
||||||
"image": "gcr.io/google-containers/k8s-cloud-kms-plugin:v0.1.1",
|
|
||||||
"command": ["/k8s-cloud-kms-plugin", "--key-uri={{kms_key_uri}}", "--path-to-unix-socket={{kms_path_to_socket}}", "--gce-config={{gce_conf_path}}", "--logtostderr", "2>\&1"],
|
|
||||||
"livenessProbe": { "httpGet": {"host": "127.0.0.1", "port": 8081, "path": "/healthz"}, "initialDelaySeconds": 3, "timeoutSeconds": 3},
|
|
||||||
"ports":[{ "name": "healthz", "containerPort": 8081, "hostPort": 8081}, { "name": "metrics", "containerPort": 8082, "hostPort": 8082}],
|
|
||||||
"volumeMounts": [{{cloud_config_mount}}, {{kms_socket_mount}}]
|
|
||||||
}
|
|
@ -16,7 +16,6 @@
|
|||||||
"spec":{
|
"spec":{
|
||||||
"hostNetwork": true,
|
"hostNetwork": true,
|
||||||
"containers":[
|
"containers":[
|
||||||
{{kms_plugin_container}}
|
|
||||||
{
|
{
|
||||||
"name": "kube-apiserver",
|
"name": "kube-apiserver",
|
||||||
"image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
|
"image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
|
||||||
|
Loading…
Reference in New Issue
Block a user