feat: implements encrypt all

Signed-off-by: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com>
This commit is contained in:
Nilekh Chaudhari 2023-01-18 00:54:47 +00:00
parent a80b423351
commit 9382fab9b6
No known key found for this signature in database
GPG Key ID: 71276AA0BA8ED691
9 changed files with 1924 additions and 76 deletions

View File

@ -26,11 +26,23 @@ import (
/* /*
EncryptionConfiguration stores the complete configuration for encryption providers. EncryptionConfiguration stores the complete configuration for encryption providers.
example: It also allows the use of wildcards to specify the resources that should be encrypted.
Use '*.<group>' to encrypt all resources within a group or '*.*' to encrypt all resources.
'*.' can be used to encrypt all resource in the core group. '*.*' will encrypt all
resources, even custom resources that are added after API server start.
Use of wildcards that overlap within the same resource list or across multiple
entries are not allowed since part of the configuration would be ineffective.
Resource lists are processed in order, with earlier lists taking precedence.
Example:
kind: EncryptionConfiguration kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1 apiVersion: apiserver.config.k8s.io/v1
resources: resources:
- resources:
- events
providers:
- identity: {} # do not encrypt events even though *.* is specified below
- resources: - resources:
- secrets - secrets
- configmaps - configmaps
@ -40,6 +52,20 @@ example:
keys: keys:
- name: key1 - name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ== secret: c2VjcmV0IGlzIHNlY3VyZQ==
- resources:
- '*.apps'
providers:
- aescbc:
keys:
- name: key2
secret: c2VjcmV0IGlzIHNlY3VyZSwgb3IgaXMgaXQ/Cg==
- resources:
- '*.*'
providers:
- aescbc:
keys:
- name: key3
secret: c2VjcmV0IGlzIHNlY3VyZSwgSSB0aGluaw==
*/ */
type EncryptionConfiguration struct { type EncryptionConfiguration struct {
metav1.TypeMeta metav1.TypeMeta
@ -50,10 +76,13 @@ type EncryptionConfiguration struct {
// ResourceConfiguration stores per resource configuration. // ResourceConfiguration stores per resource configuration.
type ResourceConfiguration struct { type ResourceConfiguration struct {
// resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource. // resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource.
// eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas) // eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas.
// Use '*.*' to encrypt all resources and '*.<group>' to encrypt all resources in a specific group.
// eg: '*.awesome.bears.example' will encrypt all resources in the group 'awesome.bears.example'.
// eg: '*.' will encrypt all resources in the core group (such as pods, configmaps, etc).
Resources []string Resources []string
// providers is a list of transformers to be used for reading and writing the resources to disk. // providers is a list of transformers to be used for reading and writing the resources to disk.
// eg: aesgcm, aescbc, secretbox, identity. // eg: aesgcm, aescbc, secretbox, identity, kms.
Providers []ProviderConfiguration Providers []ProviderConfiguration
} }

View File

@ -26,11 +26,23 @@ import (
/* /*
EncryptionConfiguration stores the complete configuration for encryption providers. EncryptionConfiguration stores the complete configuration for encryption providers.
example: It also allows the use of wildcards to specify the resources that should be encrypted.
Use '*.<group>' to encrypt all resources within a group or '*.*' to encrypt all resources.
'*.' can be used to encrypt all resource in the core group. '*.*' will encrypt all
resources, even custom resources that are added after API server start.
Use of wildcards that overlap within the same resource list or across multiple
entries are not allowed since part of the configuration would be ineffective.
Resource lists are processed in order, with earlier lists taking precedence.
Example:
kind: EncryptionConfiguration kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1 apiVersion: apiserver.config.k8s.io/v1
resources: resources:
- resources:
- events
providers:
- identity: {} # do not encrypt events even though *.* is specified below
- resources: - resources:
- secrets - secrets
- configmaps - configmaps
@ -40,6 +52,20 @@ example:
keys: keys:
- name: key1 - name: key1
secret: c2VjcmV0IGlzIHNlY3VyZQ== secret: c2VjcmV0IGlzIHNlY3VyZQ==
- resources:
- '*.apps'
providers:
- aescbc:
keys:
- name: key2
secret: c2VjcmV0IGlzIHNlY3VyZSwgb3IgaXMgaXQ/Cg==
- resources:
- '*.*'
providers:
- aescbc:
keys:
- name: key3
secret: c2VjcmV0IGlzIHNlY3VyZSwgSSB0aGluaw==
*/ */
type EncryptionConfiguration struct { type EncryptionConfiguration struct {
metav1.TypeMeta metav1.TypeMeta
@ -50,10 +76,13 @@ type EncryptionConfiguration struct {
// ResourceConfiguration stores per resource configuration. // ResourceConfiguration stores per resource configuration.
type ResourceConfiguration struct { type ResourceConfiguration struct {
// resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource. // resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource.
// eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas) // eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas.
// Use '*.*' to encrypt all resources and '*.<group>' to encrypt all resources in a specific group.
// eg: '*.awesome.bears.example' will encrypt all resources in the group 'awesome.bears.example'.
// eg: '*.' will encrypt all resources in the core group (such as pods, configmaps, etc).
Resources []string `json:"resources"` Resources []string `json:"resources"`
// providers is a list of transformers to be used for reading and writing the resources to disk. // providers is a list of transformers to be used for reading and writing the resources to disk.
// eg: aesgcm, aescbc, secretbox, identity. // eg: aesgcm, aescbc, secretbox, identity, kms.
Providers []ProviderConfiguration `json:"providers"` Providers []ProviderConfiguration `json:"providers"`
} }

View File

@ -23,6 +23,7 @@ import (
"net/url" "net/url"
"strings" "strings"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/apis/config" "k8s.io/apiserver/pkg/apis/config"
@ -34,7 +35,7 @@ const (
unsupportedSchemeErrFmt = "unsupported scheme %q for KMS provider, only unix is supported" unsupportedSchemeErrFmt = "unsupported scheme %q for KMS provider, only unix is supported"
unsupportedKMSAPIVersionErrFmt = "unsupported apiVersion %s for KMS provider, only v1 and v2 are supported" unsupportedKMSAPIVersionErrFmt = "unsupported apiVersion %s for KMS provider, only v1 and v2 are supported"
atLeastOneRequiredErrFmt = "at least one %s is required" atLeastOneRequiredErrFmt = "at least one %s is required"
invalidURLErrFmt = "invalid endpoint for kms provider, error: parse %s: net/url: invalid control character in URL" invalidURLErrFmt = "invalid endpoint for kms provider, error: %v"
mandatoryFieldErrFmt = "%s is a mandatory field for a %s" mandatoryFieldErrFmt = "%s is a mandatory field for a %s"
base64EncodingErr = "secrets must be base64 encoded" base64EncodingErr = "secrets must be base64 encoded"
zeroOrNegativeErrFmt = "%s should be a positive value" zeroOrNegativeErrFmt = "%s should be a positive value"
@ -42,6 +43,14 @@ const (
encryptionConfigNilErr = "EncryptionConfiguration can't be nil" encryptionConfigNilErr = "EncryptionConfiguration can't be nil"
invalidKMSConfigNameErrFmt = "invalid KMS provider name %s, must not contain ':'" invalidKMSConfigNameErrFmt = "invalid KMS provider name %s, must not contain ':'"
duplicateKMSConfigNameErrFmt = "duplicate KMS provider name %s, names must be unique" duplicateKMSConfigNameErrFmt = "duplicate KMS provider name %s, names must be unique"
eventsGroupErr = "'*.events.k8s.io' objects are stored using the 'events' API group in etcd. Use 'events' instead in the config file"
extensionsGroupErr = "'extensions' group has been removed and cannot be used for encryption"
starResourceErr = "use '*.' to encrypt all the resources from core API group or *.* to encrypt all resources"
overlapErr = "using overlapping resources such as 'secrets' and '*.' in the same resource list is not allowed as they will be masked"
nonRESTAPIResourceErr = "resources which do not have REST API/s cannot be encrypted"
resourceNameErr = "resource name should not contain capital letters"
resourceAcrossGroupErr = "encrypting the same resource across groups is not supported"
duplicateResourceErr = "the same resource cannot be specified multiple times"
) )
var ( var (
@ -59,7 +68,7 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
allErrs := field.ErrorList{} allErrs := field.ErrorList{}
if c == nil { if c == nil {
allErrs = append(allErrs, field.Required(root, "EncryptionConfiguration can't be nil")) allErrs = append(allErrs, field.Required(root, encryptionConfigNilErr))
return allErrs return allErrs
} }
@ -78,6 +87,9 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
allErrs = append(allErrs, field.Required(r, fmt.Sprintf(atLeastOneRequiredErrFmt, r))) allErrs = append(allErrs, field.Required(r, fmt.Sprintf(atLeastOneRequiredErrFmt, r)))
} }
allErrs = append(allErrs, validateResourceOverlap(conf.Resources, r)...)
allErrs = append(allErrs, validateResourceNames(conf.Resources, r)...)
if len(conf.Providers) == 0 { if len(conf.Providers) == 0 {
allErrs = append(allErrs, field.Required(p, fmt.Sprintf(atLeastOneRequiredErrFmt, p))) allErrs = append(allErrs, field.Required(p, fmt.Sprintf(atLeastOneRequiredErrFmt, p)))
} }
@ -103,6 +115,175 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b
return allErrs return allErrs
} }
var anyGroupAnyResource = schema.GroupResource{
Group: "*",
Resource: "*",
}
func validateResourceOverlap(resources []string, fieldPath *field.Path) field.ErrorList {
if len(resources) < 2 { // cannot have overlap with a single resource
return nil
}
var allErrs field.ErrorList
r := make([]schema.GroupResource, 0, len(resources))
for _, resource := range resources {
r = append(r, schema.ParseGroupResource(resource))
}
var hasOverlap, hasDuplicate bool
for i, r1 := range r {
for j, r2 := range r {
if i == j {
continue
}
if r1 == r2 && !hasDuplicate {
hasDuplicate = true
continue
}
if hasOverlap {
continue
}
if r1 == anyGroupAnyResource {
hasOverlap = true
continue
}
if r1.Group != r2.Group {
continue
}
if r1.Resource == "*" || r2.Resource == "*" {
hasOverlap = true
continue
}
}
}
if hasDuplicate {
allErrs = append(
allErrs,
field.Invalid(
fieldPath,
resources,
duplicateResourceErr,
),
)
}
if hasOverlap {
allErrs = append(
allErrs,
field.Invalid(
fieldPath,
resources,
overlapErr,
),
)
}
return allErrs
}
func validateResourceNames(resources []string, fieldPath *field.Path) field.ErrorList {
var allErrs field.ErrorList
for j, res := range resources {
jj := fieldPath.Index(j)
// check if resource name has capital letters
if hasCapital(res) {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
resourceNameErr,
),
)
continue
}
// check if resource is '*'
if res == "*" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
starResourceErr,
),
)
continue
}
// check if resource is:
// 'apiserveripinfo' OR
// 'serviceipallocations' OR
// 'servicenodeportallocations' OR
if res == "apiserveripinfo" ||
res == "serviceipallocations" ||
res == "servicenodeportallocations" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
nonRESTAPIResourceErr,
),
)
continue
}
// check if group is 'events.k8s.io'
gr := schema.ParseGroupResource(res)
if gr.Group == "events.k8s.io" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
eventsGroupErr,
),
)
continue
}
// check if group is 'extensions'
if gr.Group == "extensions" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
extensionsGroupErr,
),
)
continue
}
// disallow resource.* as encrypting the same resource across groups does not make sense
if gr.Group == "*" && gr.Resource != "*" {
allErrs = append(
allErrs,
field.Invalid(
jj,
resources[j],
resourceAcrossGroupErr,
),
)
continue
}
}
return allErrs
}
func validateSingleProvider(provider config.ProviderConfiguration, fieldPath *field.Path) field.ErrorList { func validateSingleProvider(provider config.ProviderConfiguration, fieldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{} allErrs := field.ErrorList{}
found := 0 found := 0
@ -225,7 +406,7 @@ func validateKMSEndpoint(c *config.KMSConfiguration, fieldPath *field.Path) fiel
u, err := url.Parse(c.Endpoint) u, err := url.Parse(c.Endpoint)
if err != nil { if err != nil {
return append(allErrs, field.Invalid(fieldPath, c.Endpoint, fmt.Sprintf("invalid endpoint for kms provider, error: %v", err))) return append(allErrs, field.Invalid(fieldPath, c.Endpoint, fmt.Sprintf(invalidURLErrFmt, err)))
} }
if u.Scheme != "unix" { if u.Scheme != "unix" {
@ -265,3 +446,7 @@ func validateKMSConfigName(c *config.KMSConfiguration, fieldPath *field.Path, km
return allErrs return allErrs
} }
func hasCapital(input string) bool {
return strings.ToLower(input) != input
}

View File

@ -22,6 +22,7 @@ import (
"time" "time"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/validation/field"
@ -370,6 +371,737 @@ func TestStructure(t *testing.T) {
"foo", fmt.Sprintf(duplicateKMSConfigNameErrFmt, "foo")), "foo", fmt.Sprintf(duplicateKMSConfigNameErrFmt, "foo")),
}, },
}, },
{
desc: "config should error when events.k8s.io group is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"events.events.k8s.io",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"events.events.k8s.io",
eventsGroupErr,
),
},
}, {
desc: "config should error when events.k8s.io group is used later in the list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
{
Resources: []string{
"secret",
"events.events.k8s.io",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(1).Child("resources").Index(1),
"events.events.k8s.io",
eventsGroupErr,
),
},
},
{
desc: "config should error when *.events.k8s.io group is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.events.k8s.io",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"*.events.k8s.io",
eventsGroupErr,
),
},
},
{
desc: "config should error when extensions group is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.extensions",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"*.extensions",
extensionsGroupErr,
),
},
},
{
desc: "config should error when foo.extensions group is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"foo.extensions",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"foo.extensions",
extensionsGroupErr,
),
},
},
{
desc: "config should error when '*' resource is used",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"*",
starResourceErr,
),
},
},
{
desc: "should error when resource name has capital letters",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"apiServerIPInfo",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"apiServerIPInfo",
resourceNameErr,
),
},
},
{
desc: "should error when resource name is apiserveripinfo",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"apiserveripinfo",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"apiserveripinfo",
nonRESTAPIResourceErr,
),
},
},
{
desc: "should error when resource name is serviceipallocations",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"serviceipallocations",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"serviceipallocations",
nonRESTAPIResourceErr,
),
},
},
{
desc: "should error when resource name is servicenodeportallocations",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"servicenodeportallocations",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(0),
"servicenodeportallocations",
nonRESTAPIResourceErr,
),
},
},
{
desc: "should not error when '*.apps' and '*.' are used within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.apps",
"*.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{},
},
{
desc: "should error when the same resource across groups is encrypted",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.",
"foos.*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources").Index(1),
"foos.*",
resourceAcrossGroupErr,
),
},
},
{
desc: "should error when secrets are specified twice within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"secrets",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"secrets",
},
duplicateResourceErr,
),
},
},
{
desc: "should error once when secrets are specified many times within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"secrets",
"secrets",
"secrets",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"secrets",
"secrets",
"secrets",
},
duplicateResourceErr,
),
},
},
{
desc: "should error when secrets are specified twice within the same resource list, via dot",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"secrets.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"secrets.",
},
duplicateResourceErr,
),
},
},
{
desc: "should error when '*.apps' and '*.' and '*.*' are used within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.apps",
"*.",
"*.*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"*.apps",
"*.",
"*.*",
},
overlapErr,
),
},
},
{
desc: "should not error when deployments.apps are specified with '*.' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"deployments.apps",
"*.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{},
},
{
desc: "should error when deployments.apps are specified with '*.apps' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"deployments.apps",
"*.apps",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"deployments.apps",
"*.apps",
},
overlapErr,
),
},
},
{
desc: "should error when secrets are specified with '*.' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"*.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"*.",
},
overlapErr,
),
},
},
{
desc: "should error when pods are specified with '*.' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"pods",
"*.",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"pods",
"*.",
},
overlapErr,
),
},
},
{
desc: "should error when other resources are specified with '*.*' within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"secrets",
"*.*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"secrets",
"*.*",
},
overlapErr,
),
},
},
{
desc: "should error when both '*.' and '*.*' are used within the same resource list",
in: &config.EncryptionConfiguration{
Resources: []config.ResourceConfiguration{
{
Resources: []string{
"*.",
"*.*",
},
Providers: []config.ProviderConfiguration{
{
KMS: &config.KMSConfiguration{
Name: "foo",
Endpoint: "unix:///tmp/kms-provider.socket",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
CacheSize: &cacheSize,
APIVersion: "v1",
},
},
},
},
},
},
reload: false,
want: field.ErrorList{
field.Invalid(
root.Index(0).Child("resources"),
[]string{
"*.",
"*.*",
},
overlapErr,
),
},
},
} }
for _, tt := range testCases { for _, tt := range testCases {
@ -505,7 +1237,7 @@ func TestKMSEndpoint(t *testing.T) {
desc: "invalid url", desc: "invalid url",
in: &config.KMSConfiguration{Endpoint: "unix:///foo\n.socket"}, in: &config.KMSConfiguration{Endpoint: "unix:///foo\n.socket"},
want: field.ErrorList{ want: field.ErrorList{
field.Invalid(endpointField, "unix:///foo\n.socket", fmt.Sprintf(invalidURLErrFmt, `"unix:///foo\n.socket"`)), field.Invalid(endpointField, "unix:///foo\n.socket", fmt.Sprintf(invalidURLErrFmt, `parse "unix:///foo\n.socket": net/url: invalid control character in URL`)),
}, },
}, },
} }

View File

@ -218,8 +218,22 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis
for _, resource := range resourceConfig.Resources { for _, resource := range resourceConfig.Resources {
resource := resource resource := resource
gr := schema.ParseGroupResource(resource) gr := schema.ParseGroupResource(resource)
resourceToPrefixTransformer[gr] = append(
resourceToPrefixTransformer[gr], transformers...) // check if resource is masked by *.group rule
anyResourceInGroup := schema.GroupResource{Group: gr.Group, Resource: "*"}
if _, masked := resourceToPrefixTransformer[anyResourceInGroup]; masked {
// an earlier rule already configured a transformer for *.group, masking this rule
// return error since this is not allowed
return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyResourceInGroup))
}
if _, masked := resourceToPrefixTransformer[anyGroupAnyResource]; masked {
// an earlier rule already configured a transformer for *.*, masking this rule
// return error since this is not allowed
return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyGroupAnyResource))
}
resourceToPrefixTransformer[gr] = append(resourceToPrefixTransformer[gr], transformers...)
} }
probes = append(probes, p...) probes = append(probes, p...)
@ -777,10 +791,34 @@ func (s StaticTransformers) TransformerForResource(resource schema.GroupResource
return transformerFromOverrides(s, resource) return transformerFromOverrides(s, resource)
} }
func transformerFromOverrides(transformerOverrides map[schema.GroupResource]value.Transformer, resource schema.GroupResource) value.Transformer { var anyGroupAnyResource = schema.GroupResource{
transformer := transformerOverrides[resource] Group: "*",
if transformer == nil { Resource: "*",
return identity.NewEncryptCheckTransformer() }
}
return transformer func transformerFromOverrides(transformerOverrides map[schema.GroupResource]value.Transformer, resource schema.GroupResource) value.Transformer {
if transformer := transformerOverrides[resource]; transformer != nil {
return transformer
}
if transformer := transformerOverrides[schema.GroupResource{
Group: resource.Group,
Resource: "*",
}]; transformer != nil {
return transformer
}
if transformer := transformerOverrides[anyGroupAnyResource]; transformer != nil {
return transformer
}
return identity.NewEncryptCheckTransformer()
}
func grYAMLString(gr schema.GroupResource) string {
if gr.Group == "" && gr.Resource == "*" {
return "*."
}
return gr.String()
} }

View File

@ -21,6 +21,7 @@ import (
"context" "context"
"encoding/base64" "encoding/base64"
"errors" "errors"
"reflect"
"strings" "strings"
"sync" "sync"
"testing" "testing"
@ -41,6 +42,7 @@ import (
"k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/legacyregistry"
"k8s.io/component-base/metrics/testutil" "k8s.io/component-base/metrics/testutil"
kmsservice "k8s.io/kms/pkg/service" kmsservice "k8s.io/kms/pkg/service"
"k8s.io/utils/pointer"
) )
const ( const (
@ -646,6 +648,584 @@ func TestKMSPluginHealthz(t *testing.T) {
} }
} }
// tests for masking rules
func TestWildcardMasking(t *testing.T) {
testCases := []struct {
desc string
config *apiserverconfig.EncryptionConfiguration
expectedError string
}{
{
desc: "resources masked by *. group",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"secrets\" is masked by earlier rule \"*.\"",
},
{
desc: "*. masked by *. group",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms2",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.\" is masked by earlier rule \"*.\"",
},
{
desc: "*.foo masked by *.foo",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"*.foo",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.foo",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms2",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.foo\" is masked by earlier rule \"*.foo\"",
},
{
desc: "*.* masked by *.*",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms2",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.*\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources masked by *. group in multiple configurations",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/another-testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"secrets\" is masked by earlier rule \"*.\"",
},
{
desc: "resources masked by *.*",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.*",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"secrets\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources masked by *.* in multiple configurations",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/another-testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"secrets\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources *. masked by *.*",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.*",
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources *. masked by *.* in multiple configurations",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/another-testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
expectedError: "resource \"*.\" is masked by earlier rule \"*.*\"",
},
{
desc: "resources not masked by any rule",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"secrets",
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
},
{
desc: "resources not masked by any rule in multiple configurations",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/another-testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
_, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config)
if errString(err) != tc.expectedError {
t.Errorf("expected error %s but got %s", tc.expectedError, errString(err))
}
})
}
}
func TestWildcardStructure(t *testing.T) {
testCases := []struct {
desc string
expectedResourceTransformers map[string]string
config *apiserverconfig.EncryptionConfiguration
errorValue string
}{
{
desc: "should not result in error",
expectedResourceTransformers: map[string]string{
"configmaps": "k8s:enc:kms:v1:kms:",
"secrets": "k8s:enc:kms:v1:another-kms:",
"events": "k8s:enc:kms:v1:fancy:",
"deployments.apps": "k8s:enc:kms:v1:kms:",
"pods": "k8s:enc:kms:v1:fancy:",
"pandas": "k8s:enc:kms:v1:fancy:",
"pandas.bears": "k8s:enc:kms:v1:yet-another-provider:",
"jobs.apps": "k8s:enc:kms:v1:kms:",
},
errorValue: "",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.apps",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "another-kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
{
Identity: &apiserverconfig.IdentityConfiguration{},
},
},
},
{
Resources: []string{
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "fancy",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "yet-another-provider",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
},
},
},
{
desc: "should result in error",
errorValue: "resource \"secrets\" is masked by earlier rule \"*.\"",
config: &apiserverconfig.EncryptionConfiguration{
Resources: []apiserverconfig.ResourceConfiguration{
{
Resources: []string{
"configmaps",
"*.",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
},
},
{
Resources: []string{
"*.*",
"secrets",
},
Providers: []apiserverconfig.ProviderConfiguration{
{
KMS: &apiserverconfig.KMSConfiguration{
Name: "kms",
APIVersion: "v1",
Timeout: &metav1.Duration{Duration: 3 * time.Second},
Endpoint: "unix:///tmp/testprovider.sock",
CacheSize: pointer.Int32(10),
},
},
{
Identity: &apiserverconfig.IdentityConfiguration{},
},
},
},
},
},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config)
if errString(err) != tc.errorValue {
t.Errorf("expected error %s but got %s", tc.errorValue, errString(err))
}
if len(tc.errorValue) > 0 {
return
}
// check if expectedResourceTransformers are present
for resource, expectedTransformerName := range tc.expectedResourceTransformers {
transformer := transformerFromOverrides(transformers, schema.ParseGroupResource(resource))
transformerName := string(
reflect.ValueOf(transformer).Elem().FieldByName("transformers").Index(0).FieldByName("Prefix").Bytes(),
)
if transformerName != expectedTransformerName {
t.Errorf("resource %s: expected same transformer name but got %v", resource, cmp.Diff(transformerName, expectedTransformerName))
}
}
})
}
}
func TestKMSPluginHealthzTTL(t *testing.T) { func TestKMSPluginHealthzTTL(t *testing.T) {
ctx := testContext(t) ctx := testContext(t)

View File

@ -1,64 +1,8 @@
apiVersion: apiserver.config.k8s.io/v1 apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration kind: EncryptionConfiguration
resources: resources:
# The set of resources here are configured using output from "kubectl api-resources -o name" in a
# kind cluster running the latest built release.
- resources: - resources:
- bindings - '*.*'
- componentstatuses
- configmaps
- endpoints
- events
- limitranges
- namespaces
- nodes
- persistentvolumeclaims
- persistentvolumes
- pods
- podtemplates
- replicationcontrollers
- resourcequotas
- secrets
- serviceaccounts
- services
- mutatingwebhookconfigurations.admissionregistration.k8s.io
- validatingwebhookconfigurations.admissionregistration.k8s.io
- customresourcedefinitions.apiextensions.k8s.io
- apiservices.apiregistration.k8s.io
- controllerrevisions.apps
- daemonsets.apps
- deployments.apps
- replicasets.apps
- statefulsets.apps
- tokenreviews.authentication.k8s.io
- localsubjectaccessreviews.authorization.k8s.io
- selfsubjectaccessreviews.authorization.k8s.io
- selfsubjectrulesreviews.authorization.k8s.io
- subjectaccessreviews.authorization.k8s.io
- horizontalpodautoscalers.autoscaling
- cronjobs.batch
- jobs.batch
- certificatesigningrequests.certificates.k8s.io
- leases.coordination.k8s.io
- endpointslices.discovery.k8s.io
- events.events.k8s.io
- flowschemas.flowcontrol.apiserver.k8s.io
- prioritylevelconfigurations.flowcontrol.apiserver.k8s.io
- ingressclasses.networking.k8s.io
- ingresses.networking.k8s.io
- networkpolicies.networking.k8s.io
- runtimeclasses.node.k8s.io
- poddisruptionbudgets.policy
- clusterrolebindings.rbac.authorization.k8s.io
- clusterroles.rbac.authorization.k8s.io
- rolebindings.rbac.authorization.k8s.io
- roles.rbac.authorization.k8s.io
- priorityclasses.scheduling.k8s.io
- csidrivers.storage.k8s.io
- csinodes.storage.k8s.io
- csistoragecapacities.storage.k8s.io
- storageclasses.storage.k8s.io
- volumeattachments.storage.k8s.io
providers: providers:
- kms: - kms:
apiVersion: v2 apiVersion: v2

View File

@ -34,13 +34,21 @@ import (
"testing" "testing"
"time" "time"
clientv3 "go.etcd.io/etcd/client/v3"
"golang.org/x/crypto/cryptobyte" "golang.org/x/crypto/cryptobyte"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/storage/value" "k8s.io/apiserver/pkg/storage/value"
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1" mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/dynamic"
featuregatetesting "k8s.io/component-base/featuregate/testing"
kmsapi "k8s.io/kms/apis/v1beta1" kmsapi "k8s.io/kms/apis/v1beta1"
"k8s.io/kubernetes/test/integration"
"k8s.io/kubernetes/test/integration/etcd"
) )
const ( const (
@ -450,7 +458,6 @@ resources:
} }
rawConfigmapEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace)) rawConfigmapEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace))
if err != nil { if err != nil {
t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace), err) t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace), err)
} }
@ -533,6 +540,234 @@ resources:
} }
} }
func TestEncryptAll(t *testing.T) {
encryptionConfig := `
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- '*.*'
providers:
- kms:
name: encrypt-all-kms-provider
cachesize: 1000
endpoint: unix:///@encrypt-all-kms-provider.sock
`
t.Run("encrypt all resources", func(t *testing.T) {
pluginMock, err := mock.NewBase64Plugin("@encrypt-all-kms-provider.sock")
if err != nil {
t.Fatalf("failed to create mock of KMS Plugin: %v", err)
}
go pluginMock.Start()
if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil {
t.Fatalf("Failed start plugin, err: %v", err)
}
defer pluginMock.CleanUp()
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)()
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)()
test, err := newTransformTest(t, encryptionConfig, false, "")
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig")
}
defer test.cleanUp()
_, serverResources, err := test.restClient.Discovery().ServerGroupsAndResources()
if err != nil {
t.Fatal(err)
}
resources := etcd.GetResources(t, serverResources)
client := dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig)
etcdStorageData := etcd.GetEtcdStorageDataForNamespace(testNamespace)
for _, resource := range resources {
gvr := resource.Mapping.Resource
stub := etcdStorageData[gvr].Stub
// continue if stub is empty
if stub == "" {
t.Errorf("skipping resource %s because stub is empty", gvr)
continue
}
dynamicClient, obj, err := etcd.JSONToUnstructured(stub, testNamespace, &meta.RESTMapping{
Resource: gvr,
GroupVersionKind: gvr.GroupVersion().WithKind(resource.Mapping.GroupVersionKind.Kind),
Scope: resource.Mapping.Scope,
}, client)
if err != nil {
t.Fatal(err)
}
_, err = dynamicClient.Create(context.TODO(), obj, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
}
}
rawClient, etcdClient, err := integration.GetEtcdClients(test.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport)
if err != nil {
t.Fatalf("failed to create etcd client: %v", err)
}
// kvClient is a wrapper around rawClient and to avoid leaking goroutines we need to
// close the client (which we can do by closing rawClient).
defer rawClient.Close()
response, err := etcdClient.Get(context.TODO(), "/"+test.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Prefix, clientv3.WithPrefix())
if err != nil {
t.Fatalf("failed to retrieve secret from etcd %v", err)
}
// assert that total key values in response in greater than 0
if len(response.Kvs) == 0 {
t.Fatalf("expected total number of keys to be greater than 0, but got %d", len(response.Kvs))
}
// assert that total response keys are greater or equal to total resources
if len(response.Kvs) < len(resources) {
t.Fatalf("expected total number of keys to be greater or equal to total resources, but got %d", len(response.Kvs))
}
wantPrefix := "k8s:enc:kms:v1:encrypt-all-kms-provider:"
for _, kv := range response.Kvs {
// the following resources are not encrypted as they are not REST APIs and hence are not expected
// to be encrypted because it would be impossible to perform a storage migration on them
if strings.Contains(kv.String(), "masterleases") ||
strings.Contains(kv.String(), "serviceips") ||
strings.Contains(kv.String(), "servicenodeports") {
// assert that these resources are not encrypted with any provider
if bytes.HasPrefix(kv.Value, []byte("k8s:enc:")) {
t.Errorf("expected resource %s to not be prefixed with %s, but got %s", kv.Key, "k8s:enc:", kv.Value)
}
continue
}
// assert that all other resources are encrypted
if !bytes.HasPrefix(kv.Value, []byte(wantPrefix)) {
t.Errorf("expected resource %s to be prefixed with %s, but got %s", kv.Key, wantPrefix, kv.Value)
}
}
})
}
func TestEncryptAllWithWildcard(t *testing.T) {
encryptionConfig := `
kind: EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
resources:
- resources:
- configmaps
providers:
- identity: {}
- resources:
- '*.batch'
providers:
- kms:
name: kms-provider
cachesize: 1000
endpoint: unix:///@kms-provider.sock
- resources:
- '*.*'
providers:
- kms:
name: encrypt-all-kms-provider
cachesize: 1000
endpoint: unix:///@encrypt-all-kms-provider.sock
`
pluginMock, err := mock.NewBase64Plugin("@kms-provider.sock")
if err != nil {
t.Fatalf("failed to create mock of KMS Plugin: %v", err)
}
go pluginMock.Start()
if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil {
t.Fatalf("Failed start plugin, err: %v", err)
}
defer pluginMock.CleanUp()
encryptAllPluginMock, err := mock.NewBase64Plugin("@encrypt-all-kms-provider.sock")
if err != nil {
t.Fatalf("failed to create mock of KMS Plugin: %v", err)
}
go encryptAllPluginMock.Start()
if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil {
t.Fatalf("Failed start plugin, err: %v", err)
}
defer encryptAllPluginMock.CleanUp()
test, err := newTransformTest(t, encryptionConfig, false, "")
if err != nil {
t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err)
}
defer test.cleanUp()
wantPrefix := "k8s:enc:kms:v1:kms-provider:"
wantPrefixForEncryptAll := "k8s:enc:kms:v1:encrypt-all-kms-provider:"
_, err = test.createJob("test-job", "default")
if err != nil {
t.Fatalf("failed to create job: %v", err)
}
rawJobsEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "jobs", "test-job", "default"))
if err != nil {
t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "jobs", "test-job", "default"), err)
}
// assert prefix for jobs
if !bytes.HasPrefix(rawJobsEnvelope.Kvs[0].Value, []byte(wantPrefix)) {
t.Fatalf("expected jobs to be prefixed with %s, but got %s", wantPrefix, rawJobsEnvelope.Kvs[0].Value)
}
_, err = test.createDeployment("test-deployment", "default")
if err != nil {
t.Fatalf("failed to create deployment: %v", err)
}
rawDeploymentsEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", "test-deployment", "default"))
if err != nil {
t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", "test-deployment", "default"), err)
}
// assert prefix for deployments
if !bytes.HasPrefix(rawDeploymentsEnvelope.Kvs[0].Value, []byte(wantPrefixForEncryptAll)) {
t.Fatalf("expected deployments to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawDeploymentsEnvelope.Kvs[0].Value)
}
test.secret, err = test.createSecret(testSecret, testNamespace)
if err != nil {
t.Fatalf("Failed to create test secret, error: %v", err)
}
rawSecretEnvelope, err := test.getRawSecretFromETCD()
if err != nil {
t.Fatalf("failed to read secrets from etcd: %v", err)
}
// assert prefix for secrets
if !bytes.HasPrefix(rawSecretEnvelope, []byte(wantPrefixForEncryptAll)) {
t.Fatalf("expected secrets to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawSecretEnvelope)
}
_, err = test.createConfigMap(testConfigmap, testNamespace)
if err != nil {
t.Fatalf("Failed to create test configmap, error: %v", err)
}
rawConfigMapEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace))
if err != nil {
t.Fatalf("failed to read configmaps from etcd: %v", err)
}
// assert configmaps do not have the encrypted data prefix
if bytes.HasPrefix(rawConfigMapEnvelope.Kvs[0].Value, []byte("k8s:enc:")) {
t.Fatalf("expected configmaps to be not encrypted, got %s", rawConfigMapEnvelope.Kvs[0].Value)
}
}
func TestEncryptionConfigHotReloadFileWatch(t *testing.T) { func TestEncryptionConfigHotReloadFileWatch(t *testing.T) {
testCases := []struct { testCases := []struct {
sleep time.Duration sleep time.Duration

View File

@ -30,6 +30,8 @@ import (
clientv3 "go.etcd.io/etcd/client/v3" clientv3 "go.etcd.io/etcd/client/v3"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -47,6 +49,7 @@ import (
"k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration"
"k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/etcd"
"k8s.io/kubernetes/test/integration/framework" "k8s.io/kubernetes/test/integration/framework"
"k8s.io/utils/pointer"
"sigs.k8s.io/yaml" "sigs.k8s.io/yaml"
) )
@ -337,6 +340,79 @@ func (e *transformTest) createConfigMap(name, namespace string) (*corev1.ConfigM
return cm, nil return cm, nil
} }
// create jobs
func (e *transformTest) createJob(name, namespace string) (*batchv1.Job, error) {
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
Image: "test",
},
},
RestartPolicy: corev1.RestartPolicyNever,
},
},
},
}
if _, err := e.restClient.BatchV1().Jobs(job.Namespace).Create(context.TODO(), job, metav1.CreateOptions{}); err != nil {
return nil, fmt.Errorf("error while creating job: %v", err)
}
return job, nil
}
// create deployment
func (e *transformTest) createDeployment(name, namespace string) (*appsv1.Deployment, error) {
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: appsv1.DeploymentSpec{
Replicas: pointer.Int32(2),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "nginx",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "nginx",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx:1.17",
Ports: []corev1.ContainerPort{
{
Name: "http",
Protocol: corev1.ProtocolTCP,
ContainerPort: 80,
},
},
},
},
},
},
},
}
if _, err := e.restClient.AppsV1().Deployments(deployment.Namespace).Create(context.TODO(), deployment, metav1.CreateOptions{}); err != nil {
return nil, fmt.Errorf("error while creating deployment: %v", err)
}
return deployment, nil
}
func gvr(group, version, resource string) schema.GroupVersionResource { func gvr(group, version, resource string) schema.GroupVersionResource {
return schema.GroupVersionResource{Group: group, Version: version, Resource: resource} return schema.GroupVersionResource{Group: group, Version: version, Resource: resource}
} }