Move challenger server inside the cluster and serve with TLS

Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>
2025-04-27 19:35:22 +00:00 · 2023-02-02 18:06:07 +02:00 · 2023-02-02 18:06:07 +02:00 · e9433d2ba7
commit e9433d2ba7
parent 7abdc7b092
8 changed files with 177 additions and 45 deletions
--- a/go.mod
+++ b/go.mod
@ -9,13 +9,14 @@ require (
 	github.com/kairos-io/kairos v1.24.3-56.0.20230118103822-e3dbd41dddd1
 	github.com/kairos-io/kcrypt v0.4.5-0.20230118125949-27183fbce7ea
 	github.com/kairos-io/tpm-helpers v0.0.0-20230119140150-3fa97128ef6b
-	github.com/mudler/go-pluggable v0.0.0-20220716112424-189d463e3ff3
+	github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5
+	github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d
 	github.com/mudler/yip v0.11.4
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/ginkgo/v2 v2.7.1
+	github.com/onsi/ginkgo/v2 v2.8.0
 	github.com/onsi/gomega v1.26.0
 	github.com/pkg/errors v0.9.1
-	github.com/spectrocloud/peg v0.0.0-20230130095319-a5208a005575
+	github.com/spectrocloud/peg v0.0.0-20230207104055-5f48d834ddf6
 	k8s.io/api v0.24.2
 	k8s.io/apimachinery v0.24.2
 	k8s.io/client-go v0.24.2
@ -95,7 +96,6 @@ require (
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
@ -123,7 +123,7 @@ require (
 	golang.org/x/crypto v0.5.0 // indirect
 	golang.org/x/net v0.5.0 // indirect
 	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/term v0.4.0 // indirect
 	golang.org/x/text v0.6.0 // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
--- a/go.sum
+++ b/go.sum
@ -619,8 +619,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/mudler/entities v0.0.0-20211108084227-d1414478861b/go.mod h1:qquFT9tYp+/NO7tTotto4BT9zSRYSMDxo2PGZwujpFA=
-github.com/mudler/go-pluggable v0.0.0-20220716112424-189d463e3ff3 h1:t4X6t8WisUy5mExfS58RBOkzaEGmuor5kOUMQS8lT2g=
-github.com/mudler/go-pluggable v0.0.0-20220716112424-189d463e3ff3/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
+github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5 h1:FaZD86+A9mVt7lh9glAryzQblMsbJYU2VnrdZ8yHlTs=
+github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
 github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d h1:/lAg9vPAAU+s35cDMCx1IyeMn+4OYfCBPqi08Q8vXDg=
 github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d/go.mod h1:HGGAOJhipApckwNV8ZTliRJqxctUv3xRY+zbQEwuytc=
 github.com/mudler/yip v0.11.4 h1:xTeSOtK2JUCCbtpkdQsZsJyIbTRwxWWBj1B/Np7bI/4=
@ -658,10 +658,10 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/ginkgo/v2 v2.1.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
-github.com/onsi/ginkgo/v2 v2.7.0 h1:/XxtEV3I3Eif/HobnVx9YmJgk8ENdRsuUmM+fLCFNow=
-github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
 github.com/onsi/ginkgo/v2 v2.7.1 h1:YgLPk+gpqDtAPeRCWEmfO8oxE6ru3xcVSXAM7wn8w9I=
 github.com/onsi/ginkgo/v2 v2.7.1/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU=
+github.com/onsi/ginkgo/v2 v2.8.0 h1:pAM+oBNPrpXRs+E/8spkeGx9QgekbRVyr74EUvRVOUI=
+github.com/onsi/ginkgo/v2 v2.8.0/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
@ -784,10 +784,8 @@ github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4S
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spectrocloud/peg v0.0.0-20230126082901-925f4b5310f4 h1:pmgQfIqkgb1svmiloglanEvj8fJ6tOwYYTyW8vXzCUg=
-github.com/spectrocloud/peg v0.0.0-20230126082901-925f4b5310f4/go.mod h1:YMaL3lRGnHWV5lfAorzH1dTwc5BL7gU0ILLYbS6elPw=
-github.com/spectrocloud/peg v0.0.0-20230130095319-a5208a005575 h1:MVc4NMEFOHojjCADp+tgGuPcMtuvyMhHe0R935Hwlt4=
-github.com/spectrocloud/peg v0.0.0-20230130095319-a5208a005575/go.mod h1:YMaL3lRGnHWV5lfAorzH1dTwc5BL7gU0ILLYbS6elPw=
+github.com/spectrocloud/peg v0.0.0-20230207104055-5f48d834ddf6 h1:21+a7wprqouCWb204xWC7kJY1iO60l1KM+/YRSsxRVk=
+github.com/spectrocloud/peg v0.0.0-20230207104055-5f48d834ddf6/go.mod h1:YMaL3lRGnHWV5lfAorzH1dTwc5BL7gU0ILLYbS6elPw=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
@ -1182,6 +1180,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
--- a/scripts/e2e-tests.sh
+++ b/scripts/e2e-tests.sh
@ -2,10 +2,11 @@

 set -e

-# This scripts prepares a cluster that runs the challenger server compiled
-# from the current checkout.
+# This scripts prepares a cluster where we install the kcrypt CRDs.
+# This is where sealed volumes are created.

 GINKGO_NODES="${GINKGO_NODES:-1}"
+K3S_IMAGE="rancher/k3s:v1.26.1-k3s1"

 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 CLUSTER_NAME=$(echo $RANDOM | md5sum | head -c 10; echo;)
@ -20,35 +21,38 @@ cleanup() {
  echo "Cleaning up $CLUSTER_NAME"
  k3d cluster delete "$CLUSTER_NAME" || true
  rm -rf "$KUBECONFIG"
-
-  # Stop the challenger server
-  kill $KMS_PID
 }
 trap cleanup EXIT

-# Create a cluster
-k3d cluster create "$CLUSTER_NAME" --image rancher/k3s:v1.26.1-k3s1
+# Create a cluster and bind ports 80 and 443 on the host
+# This will allow us to access challenger server on 10.0.2.2 which is the IP
+# on which qemu "sees" the host.
+k3d cluster create "$CLUSTER_NAME" -p '80:80@server:0' -p '443:443@server:0' --image "$K3S_IMAGE"
 k3d kubeconfig get "$CLUSTER_NAME" > "$KUBECONFIG"

+# Build the docker image
+IMG=controller:latest make docker-build
+
+# Import the image to the cluster
+k3d image import -c "$CLUSTER_NAME" controller:latest
+
 # Install cert manager
 kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml
+kubectl wait --for=condition=Available deployment --timeout=2m -n cert-manager --all

-# Install the CRDs
-kubectl apply -k "$SCRIPT_DIR/../config/crd/"
+# Replace the CLUSTER_IP in the kustomize resource
+# Only needed for debugging so that we can access the server from the host
+# (the 10.0.2.2 IP address is only useful from within qemu)
+export CLUSTER_IP=$(docker inspect "k3d-${CLUSTER_NAME}-server-0"  | jq -r '.[0].NetworkSettings.Networks[].IPAddress')
+envsubst \
+    < "$SCRIPT_DIR/../tests/assets/challenger-server-ingress.template.yaml" \
+    > "$SCRIPT_DIR/../tests/assets/challenger-server-ingress.yaml"

-# Start the challenger server locally
-CHALLENGER_PORT=$(getFreePort)
-METRICS_PORT=$(getFreePort)
-HEALTH_PROBE_PORT=$(getFreePort)
-go run "${SCRIPT_DIR}/../" \
-  --challenger-bind-address "0.0.0.0:${CHALLENGER_PORT}" \
-  --metrics-bind-address "0.0.0.0:${METRICS_PORT}" \
-  --health-probe-bind-address "0.0.0.0:${HEALTH_PROBE_PORT}" \
-  --namespace default > /dev/null 2>&1 &
-export KMS_PID=$!
+# Install the challenger server kustomization
+kubectl apply -k "$SCRIPT_DIR/../tests/assets/"

 # 10.0.2.2 is where the vm sees the host
 # https://stackoverflow.com/a/6752280
-export KMS_ADDRESS="10.0.2.2:${CHALLENGER_PORT}"
+export KMS_ADDRESS="10.0.2.2.challenger.sslip.io"

 PATH=$PATH:$GOPATH/bin ginkgo --nodes $GINKGO_NODES --fail-fast -r ./tests/
--- a/tests/assets/challenger-patch.yaml
+++ b/tests/assets/challenger-patch.yaml
@ -0,0 +1,37 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+        - name: manager
+          # Don't try to pull the image we built locally
+          imagePullPolicy: IfNotPresent
+          args:
+          - "--health-probe-bind-address"
+          - ":8081"
+          - "--metrics-bind-address"
+          - "127.0.0.1:8080"
+          - "--namespace"
+          - "default"
+          - "--leader-elect"
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kcrypt-escrow-server
+  namespace: system
+spec:
+  type: ClusterIP
+  selector:
+    control-plane: controller-manager
+  ports:
+    - name: wss
+      port: 8082
+      protocol: TCP
+      targetPort: 8082
--- a/tests/assets/challenger-server-ingress.template.yaml
+++ b/tests/assets/challenger-server-ingress.template.yaml
@ -0,0 +1,35 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: challenger-server
+  annotations:
+    cert-manager.io/cluster-issuer: "selfsigned"
+    kubernetes.io/ingress.class: "traefik"
+spec:
+  tls:
+  - hosts:
+    - 10.0.2.2.challenger.sslip.io
+    - ${CLUSTER_IP}.challenger.sslip.io
+    secretName: kms-tls
+  rules:
+    - host: 10.0.2.2.challenger.sslip.io
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: kcrypt-controller-kcrypt-escrow-server
+                port:
+                  number: 8082
+    - host: ${CLUSTER_IP}.challenger.sslip.io
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: kcrypt-controller-kcrypt-escrow-server
+                port:
+                  number: 8082
--- a/tests/assets/cluster-issuer.yaml
+++ b/tests/assets/cluster-issuer.yaml
@ -0,0 +1,8 @@
+---
+# Self-signed issuer
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: selfsigned
+spec:
+  selfSigned: {}
--- a/tests/assets/kustomization.yaml
+++ b/tests/assets/kustomization.yaml
@ -0,0 +1,16 @@
+# Adds namespace to all resources.
+namespace: default
+
+bases:
+- ../../config/default
+
+resources:
+- challenger-server-ingress.yaml
+- cluster-issuer.yaml
+
+patchesStrategicMerge:
+# Fix labels and selectors to make challenger server accessible
+- challenger-patch.yaml
+# TODO: Implement 2 ingress resources, one for http and one for https (with cert-manager cert)
+# or maybe it can be one to server both. The cert should be valid for 10.0.2.2.sslip.io
+# which is how the qemu vm will access the server.
--- a/tests/encryption_test.go
+++ b/tests/encryption_test.go
@ -6,6 +6,7 @@ import (
 	"os/exec"
 	"path"
 	"strconv"
+	"strings"
 	"syscall"

 	. "github.com/onsi/ginkgo/v2"
@ -27,10 +28,15 @@ var _ = Describe("local encrypted passphrase", func() {
 	})

 	JustBeforeEach(func() {
-		out, err := vm.Sudo(fmt.Sprintf(`cat << EOF > config.yaml
-%s
-`, config))
-		Expect(err).ToNot(HaveOccurred(), out)
+		configFile, err := os.CreateTemp("", "")
+		Expect(err).ToNot(HaveOccurred())
+		defer os.Remove(configFile.Name())
+
+		err = os.WriteFile(configFile.Name(), []byte(config), 0744)
+		Expect(err).ToNot(HaveOccurred())
+
+		err = vm.Scp(configFile.Name(), "config.yaml", "0744")
+		Expect(err).ToNot(HaveOccurred())

 		installationOutput, err = vm.Sudo("set -o pipefail && kairos-agent manual-install --device auto config.yaml 2>&1 | tee manual-install.txt")
 		Expect(err).ToNot(HaveOccurred(), installationOutput)
@ -68,7 +74,7 @@ hostname: metal-{{ trunc 4 .MachineID }}
 users:
 - name: kairos
  passwd: kairos
-EOF`
+`
 		})

 		It("boots and has an encrypted partition", func() {
@ -92,14 +98,14 @@ EOF`
 apiVersion: keyserver.kairos.io/v1alpha1
 kind: SealedVolume
 metadata:
-    name: %[1]s
-    namespace: default
+  name: "%[1]s"
+  namespace: default
 spec:
  TPMHash: "%[1]s"
  partitions:
    - label: COS_PERSISTENT
  quarantined: false
-`, tpmHash))
+`, strings.TrimSpace(tpmHash)))

 			config = fmt.Sprintf(`#cloud-config

@ -121,8 +127,7 @@ kcrypt:
    nv_index: ""
    c_index: ""
    tpm_device: ""
-
-EOF`, os.Getenv("KMS_ADDRESS"))
+`, os.Getenv("KMS_ADDRESS"))
 		})

 		AfterEach(func() {
@ -145,7 +150,7 @@ EOF`, os.Getenv("KMS_ADDRESS"))
 			)

 			secretOut, err := cmd.CombinedOutput()
-			Expect(err).ToNot(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred(), secretOut)
 			Expect(string(secretOut)).To(MatchRegexp("tpm"))
 		})
 	})
@ -207,7 +212,7 @@ kcrypt:
    c_index: ""
    tpm_device: ""

-EOF`, os.Getenv("KMS_ADDRESS"))
+`, os.Getenv("KMS_ADDRESS"))
 		})

 		AfterEach(func() {
@ -229,6 +234,33 @@ EOF`, os.Getenv("KMS_ADDRESS"))
 			Expect(out).To(MatchRegexp("/dev/mapper.*LABEL=\"COS_PERSISTENT\""), out)
 		})
 	})
+
+	When("the key management server is listening on https", func() {
+		BeforeEach(func() {
+			// TODO:
+			// - Create and ExternalNames service that points to 10.0.2.2.sslip.io (the server)
+			// - Create an ingress for the above service with a certificate generated
+			//   by cert-manager
+
+			// Create a service that points to the server running j
+			// https://github.com/traefik/traefik/issues/1816#issuecomment-322543677
+		})
+		When("the certificate is pinned on the configuration", func() {
+			It("successfully talks to the server", func() {
+				// TODO: Maybe do something simpler than installation to keep things fast?
+				// Something that proves we talked to the server.
+				// Cert should be valid for a magic domain (e.g. sslip.io). We can use
+				// cert-manager to issue one.
+			})
+		})
+		When("the certificate signed by a well known CA (system certs)", func() {
+			It("successfully talks to the server", func() {
+				// TODO: How do we get a properly signed cert? Maybe do that once,
+				// and put the cert is the assets directory?
+				// Is it possible to have a signed cert without a proper domain?
+			})
+		})
+	})
 })

 func printInstallationOutput(message string, callerSkip ...int) {