github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-01-19 08:15:01 +00:00
Code Issues Projects Releases Wiki Activity

Allow kube-apiserver to test the status of kms-plugin.

Browse Source
This commit is contained in:
immutablet
2019-05-30 11:15:35 -07:00
parent 49c34bb597
commit 05fdbb201f
15 changed files with 565 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
staging/src/k8s.io/apiserver/go.mod
View File

@@ -24,6 +24,7 @@ require (
github.com/go-openapi/swag v0.17.2 // indirect
github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
github.com/google/go-cmp v0.3.0
github.com/google/gofuzz v1.0.0
github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d
github.com/gorilla/websocket v1.4.0 // indirect

4
staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go
View File

@@ -16,7 +16,9 @@ limitations under the License.
package v1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

1
staging/src/k8s.io/apiserver/pkg/server/options/BUILD
View File

@@ -57,6 +57,7 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",

2
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/BUILD
View File

@@ -17,6 +17,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/apis/config:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/apis/config/v1:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library",
@@ -35,6 +36,7 @@ go_test(
"//staging/src/k8s.io/apiserver/pkg/apis/config:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/envelope:go_default_library",
"//vendor/github.com/google/go-cmp/cmp:go_default_library",
],
)

108
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go
View File

@@ -23,7 +23,9 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"sync"
"time"
"k8s.io/apimachinery/pkg/runtime"
@@ -31,6 +33,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/serializer"
apiserverconfig "k8s.io/apiserver/pkg/apis/config"
apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/storage/value"
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
@@ -44,8 +47,113 @@ const (
secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
kmsTransformerPrefixV1 = "k8s:enc:kms:v1:"
kmsPluginConnectionTimeout = 3 * time.Second
kmsPluginHealthzTTL = 3 * time.Second
)
type kmsPluginHealthzResponse struct {
err error
received time.Time
}
type kmsPluginProbe struct {
name string
envelope.Service
lastResponse *kmsPluginHealthzResponse
l *sync.Mutex
}
func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthzChecker {
return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
return h.Check()
})
}
// GetKMSPluginHealthzCheckers extracts KMSPluginProbes from the EncryptionConfig.
func GetKMSPluginHealthzCheckers(filepath string) ([]healthz.HealthzChecker, error) {
f, err := os.Open(filepath)
if err != nil {
return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
}
defer f.Close()
var result []healthz.HealthzChecker
probes, err := getKMSPluginProbes(f)
if err != nil {
return nil, err
}
for i, p := range probes {
probe := p
result = append(result, probe.toHealthzCheck(i))
}
return result, nil
}
func getKMSPluginProbes(reader io.Reader) ([]*kmsPluginProbe, error) {
var result []*kmsPluginProbe
configFileContents, err := ioutil.ReadAll(reader)
if err != nil {
return result, fmt.Errorf("could not read content of encryption provider configuration: %v", err)
}
config, err := loadConfig(configFileContents)
if err != nil {
return result, fmt.Errorf("error while parsing encrypiton provider configuration: %v", err)
}
for _, r := range config.Resources {
for _, p := range r.Providers {
if p.KMS != nil {
timeout := kmsPluginConnectionTimeout
if p.KMS.Timeout != nil {
if p.KMS.Timeout.Duration <= 0 {
return nil, fmt.Errorf("could not configure KMS-Plugin's probe %q, timeout should be a positive value", p.KMS.Name)
}
timeout = p.KMS.Timeout.Duration
}
s, err := envelope.NewGRPCService(p.KMS.Endpoint, timeout)
if err != nil {
return nil, fmt.Errorf("could not configure KMS-Plugin's probe %q, error: %v", p.KMS.Name, err)
}
result = append(result, &kmsPluginProbe{
name: p.KMS.Name,
Service: s,
l: &sync.Mutex{},
lastResponse: &kmsPluginHealthzResponse{},
})
}
}
}
return result, nil
}
// Check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
func (h *kmsPluginProbe) Check() error {
h.l.Lock()
defer h.l.Unlock()
if (time.Now().Sub(h.lastResponse.received)) < kmsPluginHealthzTTL {
return h.lastResponse.err
}
p, err := h.Service.Encrypt([]byte("ping"))
if err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
}
if _, err := h.Service.Decrypt(p); err != nil {
h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
}
h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
return nil
}
// GetTransformerOverrides returns the transformer overrides by reading and parsing the encryption provider configuration file
func GetTransformerOverrides(filepath string) (map[schema.GroupResource]value.Transformer, error) {
f, err := os.Open(filepath)

98
staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go
View File

@@ -24,6 +24,8 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/diff"
apiserverconfig "k8s.io/apiserver/pkg/apis/config"
@@ -507,3 +509,99 @@ resources:
})
}
}
func TestKMSPluginHealthz(t *testing.T) {
service, err := envelope.NewGRPCService("unix:///tmp/testprovider.sock", kmsPluginConnectionTimeout)
if err != nil {
t.Fatalf("Could not initialize envelopeService, error: %v", err)
}
testCases := []struct {
desc string
config string
want []*kmsPluginProbe
wantErr bool
}{
{
desc: "Install Healthz",
config: `kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- kms:
name: foo
endpoint: unix:///tmp/testprovider.sock
timeout: 15s
`,
want: []*kmsPluginProbe{
{
name: "foo",
Service: service,
},
},
},
{
desc: "Install multiple healthz",
config: `kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- kms:
name: foo
endpoint: unix:///tmp/testprovider.sock
timeout: 15s
- kms:
name: bar
endpoint: unix:///tmp/testprovider.sock
timeout: 15s
`,
want: []*kmsPluginProbe{
{
name: "foo",
Service: service,
},
{
name: "bar",
Service: service,
},
},
},
{
desc: "No KMS Providers",
config: `kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- aesgcm:
keys:
- name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ==
`,
},
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
got, err := getKMSPluginProbes(strings.NewReader(tt.config))
if err != nil && !tt.wantErr {
t.Fatalf("got %v, want nil for error", err)
}
if d := cmp.Diff(tt.want, got, cmp.Comparer(serviceComparer)); d != "" {
t.Fatalf("HealthzConfig mismatch (-want +got):\n%s", d)
}
})
}
}
// As long as got and want contain envelope.Service we will return true.
// If got has an envelope.Service and want does note (or vice versa) this will return false.
func serviceComparer(_, _ envelope.Service) bool {
return true
}

11
staging/src/k8s.io/apiserver/pkg/server/options/etcd.go
View File

@@ -31,6 +31,7 @@ import (
genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
@@ -204,6 +205,16 @@ func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
c.HealthzChecks = append(c.HealthzChecks, healthz.NamedCheck("etcd", func(r *http.Request) error {
return healthCheck()
}))
if s.EncryptionProviderConfigFilepath != "" {
kmsPluginHealthzChecks, err := encryptionconfig.GetKMSPluginHealthzCheckers(s.EncryptionProviderConfigFilepath)
if err != nil {
return err
}
c.HealthzChecks = append(c.HealthzChecks, kmsPluginHealthzChecks...)
}
return nil
}

46
staging/src/k8s.io/apiserver/pkg/server/options/etcd_test.go
View File

@@ -23,6 +23,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/storage/storagebackend"
)
@@ -194,3 +195,48 @@ func TestParseWatchCacheSizes(t *testing.T) {
})
}
}
func TestKMSHealthzEndpoint(t *testing.T) {
testCases := []struct {
name string
encryptionConfigPath string
wantChecks []string
}{
{
name: "single kms-provider, expect single kms healthz check",
encryptionConfigPath: "testdata/encryption-configs/single-kms-provider.yaml",
wantChecks: []string{"etcd", "kms-provider-0"},
},
{
name: "two kms-providers, expect two kms healthz checks",
encryptionConfigPath: "testdata/encryption-configs/multiple-kms-providers.yaml",
wantChecks: []string{"etcd", "kms-provider-0", "kms-provider-1"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
serverConfig := &server.Config{}
etcdOptions := &EtcdOptions{
EncryptionProviderConfigFilepath: tc.encryptionConfigPath,
}
if err := etcdOptions.addEtcdHealthEndpoint(serverConfig); err != nil {
t.Fatalf("Failed to add healthz error: %v", err)
}
for _, n := range tc.wantChecks {
found := false
for _, h := range serverConfig.HealthzChecks {
if n == h.Name() {
found = true
break
}
}
if !found {
t.Errorf("Missing HealthzChecker %s", n)
}
found = false
}
})
}
}

14
staging/src/k8s.io/apiserver/pkg/server/options/testdata/encryption-configs/multiple-kms-providers.yaml vendored Normal file
View File

@@ -0,0 +1,14 @@
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- kms:
name: kms-provider-1
cachesize: 1000
endpoint: unix:///@provider1.sock
- kms:
name: kms-provider-2
cachesize: 1000
endpoint: unix:///@provider2.sock

10
staging/src/k8s.io/apiserver/pkg/server/options/testdata/encryption-configs/single-kms-provider.yaml vendored Normal file
View File

@@ -0,0 +1,10 @@
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- secrets
providers:
- kms:
name: kms-provider-1
cachesize: 1000
endpoint: unix:///@kms-provider.sock