diff --git a/hack/update-internal-modules.sh b/hack/update-internal-modules.sh index 61b988c3e26..e623525ccb8 100755 --- a/hack/update-internal-modules.sh +++ b/hack/update-internal-modules.sh @@ -26,7 +26,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh" MODULES=( hack/tools staging/src/k8s.io/code-generator/examples - staging/src/k8s.io/kms/internal/plugins/mock + staging/src/k8s.io/kms/internal/plugins/_mock ) # Explicitly opt into go modules, even though we're inside a GOPATH directory diff --git a/staging/src/k8s.io/kms/internal/plugins/mock/Dockerfile b/staging/src/k8s.io/kms/internal/plugins/_mock/Dockerfile similarity index 62% rename from staging/src/k8s.io/kms/internal/plugins/mock/Dockerfile rename to staging/src/k8s.io/kms/internal/plugins/_mock/Dockerfile index adebe694081..29301ed4d55 100644 --- a/staging/src/k8s.io/kms/internal/plugins/mock/Dockerfile +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/Dockerfile @@ -20,17 +20,17 @@ WORKDIR /workspace COPY apimachinery/ apimachinery/ COPY client-go/ client-go/ COPY kms/ kms/ -WORKDIR /workspace/kms/internal/plugins/mock +WORKDIR /workspace/kms/internal/plugins/_mock ARG TARGETARCH ARG TARGETPLATFORM -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -a -o mock-kms-plugin plugin.go -RUN chmod +x mock-kms-plugin +RUN CGO_ENABLED=1 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -a -o mock-kms-plugin plugin.go -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM --platform=${TARGETPLATFORM:-linux/amd64} gcr.io/distroless/static:nonroot -WORKDIR / -COPY --from=builder /workspace/kms/internal/plugins/mock/mock-kms-plugin . +FROM alpine:latest -ENTRYPOINT [ "/mock-kms-plugin" ] +RUN apk add --update --no-cache ca-certificates gcompat +RUN apk add --no-cache softhsm + +COPY --from=builder /workspace/kms/internal/plugins/_mock/mock-kms-plugin /usr/local/bin/mock-kms-plugin + +ENTRYPOINT [ "mock-kms-plugin" ] diff --git a/staging/src/k8s.io/kms/internal/plugins/_mock/README.md b/staging/src/k8s.io/kms/internal/plugins/_mock/README.md new file mode 100644 index 00000000000..93ccf397fb5 --- /dev/null +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/README.md @@ -0,0 +1,5 @@ +# Mock KMS Plugin + +This is a mock KMS plugin for testing purposes. It implements the KMS plugin using PKCS#11 interface backed by [SoftHSM](https://www.opendnssec.org/softhsm/). It is intended to be used for testing only and not for production use. + +The directory is named `_mock` so that it is ignored by the `go mod` tooling in the root directory. diff --git a/staging/src/k8s.io/kms/internal/plugins/mock/go.mod b/staging/src/k8s.io/kms/internal/plugins/_mock/go.mod similarity index 76% rename from staging/src/k8s.io/kms/internal/plugins/mock/go.mod rename to staging/src/k8s.io/kms/internal/plugins/_mock/go.mod index 68e04178925..ade66228952 100644 --- a/staging/src/k8s.io/kms/internal/plugins/mock/go.mod +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/go.mod @@ -3,6 +3,7 @@ module k8s.io/kms/plugins/mock go 1.19 require ( + github.com/ThalesIgnite/crypto11 v1.2.5 k8s.io/klog/v2 v2.100.1 k8s.io/kms v0.0.0-00010101000000-000000000000 ) @@ -11,15 +12,15 @@ require ( github.com/go-logr/logr v1.2.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f // indirect + github.com/pkg/errors v0.8.1 // indirect + github.com/thales-e-security/pool v0.0.2 // indirect golang.org/x/net v0.13.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect - golang.org/x/time v0.3.0 // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect google.golang.org/grpc v1.54.0 // indirect google.golang.org/protobuf v1.31.0 // indirect - k8s.io/client-go v0.0.0 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect ) replace ( diff --git a/staging/src/k8s.io/kms/internal/plugins/mock/go.sum b/staging/src/k8s.io/kms/internal/plugins/_mock/go.sum similarity index 79% rename from staging/src/k8s.io/kms/internal/plugins/mock/go.sum rename to staging/src/k8s.io/kms/internal/plugins/_mock/go.sum index 4237d8303bc..edcaff71dc2 100644 --- a/staging/src/k8s.io/kms/internal/plugins/mock/go.sum +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/go.sum @@ -1,3 +1,7 @@ +github.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY+9ef8E= +github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -10,6 +14,17 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f h1:eVB9ELsoq5ouItQBr5Tj334bhPJG/MX+m7rTchmzVUQ= +github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/thales-e-security/pool v0.0.2 h1:RAPs4q2EbWsTit6tpzuvTFlgFRJ3S8Evf5gtvVDbmPg= +github.com/thales-e-security/pool v0.0.2/go.mod h1:qtpMm2+thHtqhLzTwgDBj/OuNnMpupY8mv0Phz0gjhU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -35,8 +50,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -56,4 +69,3 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/staging/src/k8s.io/kms/internal/plugins/_mock/kms.yaml b/staging/src/k8s.io/kms/internal/plugins/_mock/kms.yaml new file mode 100644 index 00000000000..f2c01f1db4d --- /dev/null +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/kms.yaml @@ -0,0 +1,69 @@ +apiVersion: v1 +kind: Pod +metadata: + name: mock-kmsv2-provider + namespace: kube-system + labels: + tier: control-plane + component: mock-kmsv2-provider +spec: + # hostNetwork: true is required because the plugin is run as a static pod + # on the control plane node and needs to run before the CNI plugins are initialized. + hostNetwork: true + initContainers: + - args: + - | + #!/bin/sh + set -e + set -x + + # if token exists, skip initialization + if [ $(ls -1 /var/lib/softhsm/tokens | wc -l) -ge 1 ]; then + echo "Skipping initialization of softhsm" + exit 0 + fi + + mkdir -p /var/lib/softhsm/tokens + apk add --update --no-cache ca-certificates jq + apk add --no-cache ccid opensc softhsm + + TOKEN_LABEL=$(jq -r '.tokenLabel' /etc/softhsm-config.json) + PIN=$(jq -r '.pin' /etc/softhsm-config.json) + MODULE_PATH=$(jq -r '.path' /etc/softhsm-config.json) + + softhsm2-util --init-token --free --label $TOKEN_LABEL --pin $PIN --so-pin $PIN + pkcs11-tool --module $MODULE_PATH --keygen --key-type aes:32 --pin $PIN --token-label $TOKEN_LABEL --label kms-test + command: + - /bin/sh + - -c + image: alpine:latest + imagePullPolicy: IfNotPresent + name: init-mock-kmsv2-provider + volumeMounts: + - mountPath: /var/lib/softhsm/tokens + name: softhsm-tokens + - mountPath: /etc/softhsm-config.json + name: softhsm-config + containers: + - name: mock-kmsv2-provider + image: localhost:5000/mock-kms-provider:e2e + imagePullPolicy: IfNotPresent + volumeMounts: + - name: sock + mountPath: /tmp + - name: softhsm-config + mountPath: /etc/softhsm-config.json + - name: softhsm-tokens + mountPath: /var/lib/softhsm/tokens + volumes: + - name: sock + hostPath: + path: /tmp + - name: softhsm-config + hostPath: + path: /etc/softhsm-config.json + type: File + - name: softhsm-tokens + hostPath: + path: /var/lib/softhsm/tokens + type: DirectoryOrCreate diff --git a/staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go b/staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go new file mode 100644 index 00000000000..8762a077e85 --- /dev/null +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go @@ -0,0 +1,118 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pkcs11 + +import ( + "context" + "crypto/cipher" + "crypto/rand" + "fmt" + + crypot11 "github.com/ThalesIgnite/crypto11" + + "k8s.io/kms/pkg/service" +) + +const ( + mockAnnotationKey = "version.encryption.remote.io" +) + +var _ service.Service = &pkcs11RemoteService{} + +type pkcs11RemoteService struct { + keyID string + aead cipher.AEAD +} + +// NewPKCS11RemoteService creates a new PKCS11 remote service with SoftHSMv2 configuration file and keyID +func NewPKCS11RemoteService(configFilePath, keyID string) (service.Service, error) { + ctx, err := crypot11.ConfigureFromFile(configFilePath) + if err != nil { + return nil, err + } + + if len(keyID) == 0 { + return nil, fmt.Errorf("invalid keyID") + } + + remoteService := &pkcs11RemoteService{ + keyID: keyID, + } + + key, err := ctx.FindKey(nil, []byte(keyID)) + if err != nil { + return nil, err + } + if key == nil { + return nil, fmt.Errorf("key not found") + } + if remoteService.aead, err = key.NewGCM(); err != nil { + return nil, err + } + + return remoteService, nil +} + +func (s *pkcs11RemoteService) Encrypt(ctx context.Context, uid string, plaintext []byte) (*service.EncryptResponse, error) { + nonceSize := s.aead.NonceSize() + result := make([]byte, nonceSize+s.aead.Overhead()+len(plaintext)) + n, err := rand.Read(result[:nonceSize]) + if err != nil { + return nil, err + } + if n != nonceSize { + return nil, fmt.Errorf("unable to read sufficient random bytes") + } + cipherText := s.aead.Seal(result[nonceSize:nonceSize], result[:nonceSize], plaintext, []byte(s.keyID)) + + return &service.EncryptResponse{ + Ciphertext: result[:nonceSize+len(cipherText)], + KeyID: s.keyID, + Annotations: map[string][]byte{ + mockAnnotationKey: []byte("1"), + }, + }, nil +} + +func (s *pkcs11RemoteService) Decrypt(ctx context.Context, uid string, req *service.DecryptRequest) ([]byte, error) { + if len(req.Annotations) != 1 { + return nil, fmt.Errorf("invalid annotations") + } + if v, ok := req.Annotations[mockAnnotationKey]; !ok || string(v) != "1" { + return nil, fmt.Errorf("invalid version in annotations") + } + + if req.KeyID != s.keyID { + return nil, fmt.Errorf("invalid keyID") + } + + nonceSize := s.aead.NonceSize() + data := req.Ciphertext + if len(data) < nonceSize { + return nil, fmt.Errorf("the stored data was shorter than the required size") + } + + return s.aead.Open(nil, data[:nonceSize], data[nonceSize:], []byte(s.keyID)) +} + +func (s *pkcs11RemoteService) Status(ctx context.Context) (*service.StatusResponse, error) { + return &service.StatusResponse{ + Version: "v2beta1", + Healthz: "ok", + KeyID: s.keyID, + }, nil +} diff --git a/staging/src/k8s.io/kms/internal/plugins/mock/plugin.go b/staging/src/k8s.io/kms/internal/plugins/_mock/plugin.go similarity index 82% rename from staging/src/k8s.io/kms/internal/plugins/mock/plugin.go rename to staging/src/k8s.io/kms/internal/plugins/_mock/plugin.go index ad9d2903d1a..370cc957795 100644 --- a/staging/src/k8s.io/kms/internal/plugins/mock/plugin.go +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/plugin.go @@ -25,14 +25,15 @@ import ( "time" "k8s.io/klog/v2" - "k8s.io/kms/internal" "k8s.io/kms/pkg/service" "k8s.io/kms/pkg/util" + "k8s.io/kms/plugins/mock/pkcs11" ) var ( - listenAddr = flag.String("listen-addr", "unix:///tmp/kms.socket", "gRPC listen address") - timeout = flag.Duration("timeout", 5*time.Second, "gRPC timeout") + listenAddr = flag.String("listen-addr", "unix:///tmp/kms.socket", "gRPC listen address") + timeout = flag.Duration("timeout", 5*time.Second, "gRPC timeout") + configFilePath = flag.String("config-file-path", "/etc/softhsm-config.json", "SoftHSM config file path") ) func main() { @@ -44,7 +45,7 @@ func main() { os.Exit(1) } - remoteKMSService, err := internal.NewMockAESService("somerandomstring", "aes-key-id") + remoteKMSService, err := pkcs11.NewPKCS11RemoteService(*configFilePath, "kms-test") if err != nil { klog.ErrorS(err, "failed to create remote service") os.Exit(1) diff --git a/staging/src/k8s.io/kms/internal/plugins/mock/kms.yaml b/staging/src/k8s.io/kms/internal/plugins/mock/kms.yaml deleted file mode 100644 index 28398cfe75f..00000000000 --- a/staging/src/k8s.io/kms/internal/plugins/mock/kms.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: mock-kmsv2-provider - namespace: kube-system - labels: - tier: control-plane - component: mock-kmsv2-provider -spec: - hostNetwork: true - containers: - - name: mock-kmsv2-provider - image: localhost:5000/mock-kms-provider:e2e - imagePullPolicy: IfNotPresent - volumeMounts: - - name: sock - mountPath: /tmp - volumes: - - name: sock - hostPath: - path: /tmp diff --git a/test/e2e/testing-manifests/auth/encrypt/README.md b/test/e2e/testing-manifests/auth/encrypt/README.md index 27fb2e7d159..343a9ba9c2b 100644 --- a/test/e2e/testing-manifests/auth/encrypt/README.md +++ b/test/e2e/testing-manifests/auth/encrypt/README.md @@ -1,6 +1,6 @@ # Encryption at rest testing manifests -This directory contains manifests for testing encryption at rest with a [mock KMS provider](../../../../../staging/src/k8s.io/kms/internal/plugins/mock). The mock KMS provider is a fake KMS provider that does not communicate with any external KMS. It is used for testing purposes only. +This directory contains manifests for testing encryption at rest with a [mock KMS provider](../../../../../staging/src/k8s.io/kms/internal/plugins/_mock). The mock KMS provider is a fake KMS provider that does not communicate with any external KMS. It is used for testing purposes only. ## run-e2e.sh diff --git a/test/e2e/testing-manifests/auth/encrypt/kind.yaml b/test/e2e/testing-manifests/auth/encrypt/kind.yaml index 94e09a5c256..621d604d4a6 100644 --- a/test/e2e/testing-manifests/auth/encrypt/kind.yaml +++ b/test/e2e/testing-manifests/auth/encrypt/kind.yaml @@ -13,7 +13,11 @@ nodes: readOnly: true propagation: None - containerPath: /etc/kubernetes/manifests/kubernetes-kms.yaml - hostPath: staging/src/k8s.io/kms/internal/plugins/mock/kms.yaml + hostPath: staging/src/k8s.io/kms/internal/plugins/_mock/kms.yaml + readOnly: true + propagation: None + - containerPath: /etc/softhsm-config.json + hostPath: test/e2e/testing-manifests/auth/encrypt/softhsm-config.json readOnly: true propagation: None kubeadmConfigPatches: diff --git a/test/e2e/testing-manifests/auth/encrypt/run-e2e.sh b/test/e2e/testing-manifests/auth/encrypt/run-e2e.sh index 3664ccb2634..071df45af72 100755 --- a/test/e2e/testing-manifests/auth/encrypt/run-e2e.sh +++ b/test/e2e/testing-manifests/auth/encrypt/run-e2e.sh @@ -36,7 +36,7 @@ build_and_push_mock_plugin() { --platform linux/amd64 \ --output=type=docker \ -t localhost:5000/mock-kms-provider:e2e \ - -f staging/src/k8s.io/kms/internal/plugins/mock/Dockerfile staging/src/k8s.io/ \ + -f staging/src/k8s.io/kms/internal/plugins/_mock/Dockerfile staging/src/k8s.io/ \ --progress=plain; docker push localhost:5000/mock-kms-provider:e2e diff --git a/test/e2e/testing-manifests/auth/encrypt/softhsm-config.json b/test/e2e/testing-manifests/auth/encrypt/softhsm-config.json new file mode 100644 index 00000000000..b0c744a0f39 --- /dev/null +++ b/test/e2e/testing-manifests/auth/encrypt/softhsm-config.json @@ -0,0 +1,5 @@ +{ + "path": "/usr/lib/softhsm/libsofthsm2.so", + "tokenLabel": "kms-test", + "pin": "kms-test" +}