[KMSv2] pkcs11 reference implementation using SoftHSM

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

hack/update-internal-modules.sh

@@ -26,7 +26,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 MODULES=(
     hack/tools
     staging/src/k8s.io/code-generator/examples
-    staging/src/k8s.io/kms/internal/plugins/mock
+    staging/src/k8s.io/kms/internal/plugins/_mock
 )
 
 # Explicitly opt into go modules, even though we're inside a GOPATH directory

staging/src/k8s.io/kms/internal/plugins/_mock/Dockerfile

@@ -20,17 +20,17 @@ WORKDIR /workspace
 COPY apimachinery/ apimachinery/
 COPY client-go/ client-go/
 COPY kms/ kms/
-WORKDIR /workspace/kms/internal/plugins/mock
+WORKDIR /workspace/kms/internal/plugins/_mock
 
 ARG TARGETARCH
 ARG TARGETPLATFORM
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -a -o mock-kms-plugin plugin.go
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -a -o mock-kms-plugin plugin.go
-RUN chmod +x mock-kms-plugin
 
-# Use distroless as minimal base image to package the manager binary
+FROM alpine:latest
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM --platform=${TARGETPLATFORM:-linux/amd64} gcr.io/distroless/static:nonroot
-WORKDIR /
-COPY --from=builder /workspace/kms/internal/plugins/mock/mock-kms-plugin .
 
-ENTRYPOINT [ "/mock-kms-plugin" ]
+RUN apk add --update --no-cache ca-certificates gcompat
+RUN apk add --no-cache softhsm
+
+COPY --from=builder /workspace/kms/internal/plugins/_mock/mock-kms-plugin /usr/local/bin/mock-kms-plugin
+
+ENTRYPOINT [ "mock-kms-plugin" ]

staging/src/k8s.io/kms/internal/plugins/_mock/README.md

@@ -0,0 +1,5 @@
+# Mock KMS Plugin
+
+This is a mock KMS plugin for testing purposes. It implements the KMS plugin using PKCS#11 interface backed by [SoftHSM](https://www.opendnssec.org/softhsm/). It is intended to be used for testing only and not for production use.
+
+The directory is named `_mock` so that it is ignored by the `go mod` tooling in the root directory.

staging/src/k8s.io/kms/internal/plugins/_mock/go.mod

@@ -3,6 +3,7 @@ module k8s.io/kms/plugins/mock
 go 1.19
 
 require (
+	github.com/ThalesIgnite/crypto11 v1.2.5
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/kms v0.0.0-00010101000000-000000000000
 )
@@ -11,15 +12,15 @@ require (
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f // indirect
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/thales-e-security/pool v0.0.2 // indirect
 	golang.org/x/net v0.13.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/text v0.11.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
 	google.golang.org/grpc v1.54.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
-	k8s.io/client-go v0.0.0 // indirect
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 )
 
 replace (

staging/src/k8s.io/kms/internal/plugins/_mock/go.sum

@@ -1,3 +1,7 @@
+github.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY+9ef8E=
+github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -10,6 +14,17 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f h1:eVB9ELsoq5ouItQBr5Tj334bhPJG/MX+m7rTchmzVUQ=
+github.com/miekg/pkcs11 v1.0.3-0.20190429190417-a667d056470f/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/thales-e-security/pool v0.0.2 h1:RAPs4q2EbWsTit6tpzuvTFlgFRJ3S8Evf5gtvVDbmPg=
+github.com/thales-e-security/pool v0.0.2/go.mod h1:qtpMm2+thHtqhLzTwgDBj/OuNnMpupY8mv0Phz0gjhU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -35,8 +50,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
 golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -56,4 +69,3 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
-k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=

staging/src/k8s.io/kms/internal/plugins/_mock/kms.yaml

@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: mock-kmsv2-provider
+  namespace: kube-system
+  labels:
+    tier: control-plane
+    component: mock-kmsv2-provider
+spec:
+  # hostNetwork: true is required because the plugin is run as a static pod
+  # on the control plane node and needs to run before the CNI plugins are initialized.
+  hostNetwork: true
+  initContainers:
+  - args:
+    - |
+      #!/bin/sh
+      set -e
+      set -x
+
+      # if token exists, skip initialization
+      if [ $(ls -1 /var/lib/softhsm/tokens | wc -l) -ge 1 ]; then
+        echo "Skipping initialization of softhsm"
+        exit 0
+      fi
+
+      mkdir -p /var/lib/softhsm/tokens
+      apk add --update --no-cache ca-certificates jq
+      apk add --no-cache ccid opensc softhsm
+      
+      TOKEN_LABEL=$(jq -r '.tokenLabel' /etc/softhsm-config.json)
+      PIN=$(jq -r '.pin' /etc/softhsm-config.json)
+      MODULE_PATH=$(jq -r '.path' /etc/softhsm-config.json)
+
+      softhsm2-util --init-token --free --label $TOKEN_LABEL --pin $PIN --so-pin $PIN
+      pkcs11-tool --module $MODULE_PATH --keygen --key-type aes:32 --pin $PIN --token-label $TOKEN_LABEL --label kms-test
+    command:
+    - /bin/sh
+    - -c
+    image: alpine:latest
+    imagePullPolicy: IfNotPresent
+    name: init-mock-kmsv2-provider
+    volumeMounts:
+    - mountPath: /var/lib/softhsm/tokens
+      name: softhsm-tokens
+    - mountPath: /etc/softhsm-config.json
+      name: softhsm-config
+  containers:
+    - name: mock-kmsv2-provider
+      image: localhost:5000/mock-kms-provider:e2e
+      imagePullPolicy: IfNotPresent
+      volumeMounts:
+        - name: sock
+          mountPath: /tmp
+        - name: softhsm-config
+          mountPath: /etc/softhsm-config.json
+        - name: softhsm-tokens
+          mountPath: /var/lib/softhsm/tokens
+  volumes:
+    - name: sock
+      hostPath:
+        path: /tmp
+    - name: softhsm-config
+      hostPath:
+        path: /etc/softhsm-config.json
+        type: File
+    - name: softhsm-tokens
+      hostPath:
+        path: /var/lib/softhsm/tokens
+        type: DirectoryOrCreate

staging/src/k8s.io/kms/internal/plugins/_mock/pkcs11/pkcs11.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pkcs11
+
+import (
+	"context"
+	"crypto/cipher"
+	"crypto/rand"
+	"fmt"
+
+	crypot11 "github.com/ThalesIgnite/crypto11"
+
+	"k8s.io/kms/pkg/service"
+)
+
+const (
+	mockAnnotationKey = "version.encryption.remote.io"
+)
+
+var _ service.Service = &pkcs11RemoteService{}
+
+type pkcs11RemoteService struct {
+	keyID string
+	aead  cipher.AEAD
+}
+
+// NewPKCS11RemoteService creates a new PKCS11 remote service with SoftHSMv2 configuration file and keyID
+func NewPKCS11RemoteService(configFilePath, keyID string) (service.Service, error) {
+	ctx, err := crypot11.ConfigureFromFile(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(keyID) == 0 {
+		return nil, fmt.Errorf("invalid keyID")
+	}
+
+	remoteService := &pkcs11RemoteService{
+		keyID: keyID,
+	}
+
+	key, err := ctx.FindKey(nil, []byte(keyID))
+	if err != nil {
+		return nil, err
+	}
+	if key == nil {
+		return nil, fmt.Errorf("key not found")
+	}
+	if remoteService.aead, err = key.NewGCM(); err != nil {
+		return nil, err
+	}
+
+	return remoteService, nil
+}
+
+func (s *pkcs11RemoteService) Encrypt(ctx context.Context, uid string, plaintext []byte) (*service.EncryptResponse, error) {
+	nonceSize := s.aead.NonceSize()
+	result := make([]byte, nonceSize+s.aead.Overhead()+len(plaintext))
+	n, err := rand.Read(result[:nonceSize])
+	if err != nil {
+		return nil, err
+	}
+	if n != nonceSize {
+		return nil, fmt.Errorf("unable to read sufficient random bytes")
+	}
+	cipherText := s.aead.Seal(result[nonceSize:nonceSize], result[:nonceSize], plaintext, []byte(s.keyID))
+
+	return &service.EncryptResponse{
+		Ciphertext: result[:nonceSize+len(cipherText)],
+		KeyID:      s.keyID,
+		Annotations: map[string][]byte{
+			mockAnnotationKey: []byte("1"),
+		},
+	}, nil
+}
+
+func (s *pkcs11RemoteService) Decrypt(ctx context.Context, uid string, req *service.DecryptRequest) ([]byte, error) {
+	if len(req.Annotations) != 1 {
+		return nil, fmt.Errorf("invalid annotations")
+	}
+	if v, ok := req.Annotations[mockAnnotationKey]; !ok || string(v) != "1" {
+		return nil, fmt.Errorf("invalid version in annotations")
+	}
+
+	if req.KeyID != s.keyID {
+		return nil, fmt.Errorf("invalid keyID")
+	}
+
+	nonceSize := s.aead.NonceSize()
+	data := req.Ciphertext
+	if len(data) < nonceSize {
+		return nil, fmt.Errorf("the stored data was shorter than the required size")
+	}
+
+	return s.aead.Open(nil, data[:nonceSize], data[nonceSize:], []byte(s.keyID))
+}
+
+func (s *pkcs11RemoteService) Status(ctx context.Context) (*service.StatusResponse, error) {
+	return &service.StatusResponse{
+		Version: "v2beta1",
+		Healthz: "ok",
+		KeyID:   s.keyID,
+	}, nil
+}

staging/src/k8s.io/kms/internal/plugins/_mock/plugin.go

@@ -25,14 +25,15 @@ import (
 	"time"
 
 	"k8s.io/klog/v2"
-	"k8s.io/kms/internal"
 	"k8s.io/kms/pkg/service"
 	"k8s.io/kms/pkg/util"
+	"k8s.io/kms/plugins/mock/pkcs11"
 )
 
 var (
-	listenAddr = flag.String("listen-addr", "unix:///tmp/kms.socket", "gRPC listen address")
+	listenAddr     = flag.String("listen-addr", "unix:///tmp/kms.socket", "gRPC listen address")
-	timeout    = flag.Duration("timeout", 5*time.Second, "gRPC timeout")
+	timeout        = flag.Duration("timeout", 5*time.Second, "gRPC timeout")
+	configFilePath = flag.String("config-file-path", "/etc/softhsm-config.json", "SoftHSM config file path")
 )
 
 func main() {
@@ -44,7 +45,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	remoteKMSService, err := internal.NewMockAESService("somerandomstring", "aes-key-id")
+	remoteKMSService, err := pkcs11.NewPKCS11RemoteService(*configFilePath, "kms-test")
 	if err != nil {
 		klog.ErrorS(err, "failed to create remote service")
 		os.Exit(1)

staging/src/k8s.io/kms/internal/plugins/mock/kms.yaml

@@ -1,21 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: mock-kmsv2-provider
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    component: mock-kmsv2-provider
-spec:
-  hostNetwork: true
-  containers:
-    - name: mock-kmsv2-provider
-      image: localhost:5000/mock-kms-provider:e2e
-      imagePullPolicy: IfNotPresent
-      volumeMounts:
-        - name: sock
-          mountPath: /tmp
-  volumes:
-    - name: sock
-      hostPath:
-        path: /tmp

test/e2e/testing-manifests/auth/encrypt/README.md

@@ -1,6 +1,6 @@
 # Encryption at rest testing manifests
 
-This directory contains manifests for testing encryption at rest with a [mock KMS provider](../../../../../staging/src/k8s.io/kms/internal/plugins/mock). The mock KMS provider is a fake KMS provider that does not communicate with any external KMS. It is used for testing purposes only.
+This directory contains manifests for testing encryption at rest with a [mock KMS provider](../../../../../staging/src/k8s.io/kms/internal/plugins/_mock). The mock KMS provider is a fake KMS provider that does not communicate with any external KMS. It is used for testing purposes only.
 
 ## run-e2e.sh
 

test/e2e/testing-manifests/auth/encrypt/kind.yaml

@@ -13,7 +13,11 @@ nodes:
     readOnly: true
     propagation: None
   - containerPath: /etc/kubernetes/manifests/kubernetes-kms.yaml
-    hostPath: staging/src/k8s.io/kms/internal/plugins/mock/kms.yaml
+    hostPath: staging/src/k8s.io/kms/internal/plugins/_mock/kms.yaml
+    readOnly: true
+    propagation: None
+  - containerPath: /etc/softhsm-config.json
+    hostPath: test/e2e/testing-manifests/auth/encrypt/softhsm-config.json
     readOnly: true
     propagation: None
   kubeadmConfigPatches:

test/e2e/testing-manifests/auth/encrypt/run-e2e.sh

@@ -36,7 +36,7 @@ build_and_push_mock_plugin() {
         --platform linux/amd64 \
         --output=type=docker \
         -t localhost:5000/mock-kms-provider:e2e \
-        -f staging/src/k8s.io/kms/internal/plugins/mock/Dockerfile staging/src/k8s.io/ \
+        -f staging/src/k8s.io/kms/internal/plugins/_mock/Dockerfile staging/src/k8s.io/ \
         --progress=plain;
 
     docker push localhost:5000/mock-kms-provider:e2e

test/e2e/testing-manifests/auth/encrypt/softhsm-config.json

@@ -0,0 +1,5 @@
+{
+    "path": "/usr/lib/softhsm/libsofthsm2.so",
+    "tokenLabel": "kms-test",
+    "pin": "kms-test"
+}