diff --git a/go.mod b/go.mod index 0d858ce..2e2ebcc 100644 --- a/go.mod +++ b/go.mod @@ -9,13 +9,14 @@ require ( github.com/kairos-io/kairos v1.24.3-56.0.20230118103822-e3dbd41dddd1 github.com/kairos-io/kcrypt v0.4.5-0.20230118125949-27183fbce7ea github.com/kairos-io/tpm-helpers v0.0.0-20230119140150-3fa97128ef6b - github.com/mudler/go-pluggable v0.0.0-20220716112424-189d463e3ff3 + github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5 + github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d github.com/mudler/yip v0.11.4 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/ginkgo/v2 v2.7.1 + github.com/onsi/ginkgo/v2 v2.8.0 github.com/onsi/gomega v1.26.0 github.com/pkg/errors v0.9.1 - github.com/spectrocloud/peg v0.0.0-20230130095319-a5208a005575 + github.com/spectrocloud/peg v0.0.0-20230207104055-5f48d834ddf6 k8s.io/api v0.24.2 k8s.io/apimachinery v0.24.2 k8s.io/client-go v0.24.2 @@ -95,7 +96,6 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/nxadm/tail v1.4.8 // indirect @@ -123,7 +123,7 @@ require ( golang.org/x/crypto v0.5.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect - golang.org/x/sys v0.4.0 // indirect + golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect diff --git a/go.sum b/go.sum index f14b7ba..3f13127 100644 --- a/go.sum +++ b/go.sum @@ -619,8 +619,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mudler/entities v0.0.0-20211108084227-d1414478861b/go.mod h1:qquFT9tYp+/NO7tTotto4BT9zSRYSMDxo2PGZwujpFA= -github.com/mudler/go-pluggable v0.0.0-20220716112424-189d463e3ff3 h1:t4X6t8WisUy5mExfS58RBOkzaEGmuor5kOUMQS8lT2g= -github.com/mudler/go-pluggable v0.0.0-20220716112424-189d463e3ff3/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI= +github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5 h1:FaZD86+A9mVt7lh9glAryzQblMsbJYU2VnrdZ8yHlTs= +github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI= github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d h1:/lAg9vPAAU+s35cDMCx1IyeMn+4OYfCBPqi08Q8vXDg= github.com/mudler/go-processmanager v0.0.0-20220724164624-c45b5c61312d/go.mod h1:HGGAOJhipApckwNV8ZTliRJqxctUv3xRY+zbQEwuytc= github.com/mudler/yip v0.11.4 h1:xTeSOtK2JUCCbtpkdQsZsJyIbTRwxWWBj1B/Np7bI/4= @@ -658,10 +658,10 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.7.0 h1:/XxtEV3I3Eif/HobnVx9YmJgk8ENdRsuUmM+fLCFNow= -github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= github.com/onsi/ginkgo/v2 v2.7.1 h1:YgLPk+gpqDtAPeRCWEmfO8oxE6ru3xcVSXAM7wn8w9I= github.com/onsi/ginkgo/v2 v2.7.1/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU= +github.com/onsi/ginkgo/v2 v2.8.0 h1:pAM+oBNPrpXRs+E/8spkeGx9QgekbRVyr74EUvRVOUI= +github.com/onsi/ginkgo/v2 v2.8.0/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -784,10 +784,8 @@ github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4S github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spectrocloud/peg v0.0.0-20230126082901-925f4b5310f4 h1:pmgQfIqkgb1svmiloglanEvj8fJ6tOwYYTyW8vXzCUg= -github.com/spectrocloud/peg v0.0.0-20230126082901-925f4b5310f4/go.mod h1:YMaL3lRGnHWV5lfAorzH1dTwc5BL7gU0ILLYbS6elPw= -github.com/spectrocloud/peg v0.0.0-20230130095319-a5208a005575 h1:MVc4NMEFOHojjCADp+tgGuPcMtuvyMhHe0R935Hwlt4= -github.com/spectrocloud/peg v0.0.0-20230130095319-a5208a005575/go.mod h1:YMaL3lRGnHWV5lfAorzH1dTwc5BL7gU0ILLYbS6elPw= +github.com/spectrocloud/peg v0.0.0-20230207104055-5f48d834ddf6 h1:21+a7wprqouCWb204xWC7kJY1iO60l1KM+/YRSsxRVk= +github.com/spectrocloud/peg v0.0.0-20230207104055-5f48d834ddf6/go.mod h1:YMaL3lRGnHWV5lfAorzH1dTwc5BL7gU0ILLYbS6elPw= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= @@ -1182,6 +1180,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/scripts/e2e-tests.sh b/scripts/e2e-tests.sh index 397993d..c82ea96 100755 --- a/scripts/e2e-tests.sh +++ b/scripts/e2e-tests.sh @@ -2,10 +2,11 @@ set -e -# This scripts prepares a cluster that runs the challenger server compiled -# from the current checkout. +# This scripts prepares a cluster where we install the kcrypt CRDs. +# This is where sealed volumes are created. GINKGO_NODES="${GINKGO_NODES:-1}" +K3S_IMAGE="rancher/k3s:v1.26.1-k3s1" SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) CLUSTER_NAME=$(echo $RANDOM | md5sum | head -c 10; echo;) @@ -20,35 +21,38 @@ cleanup() { echo "Cleaning up $CLUSTER_NAME" k3d cluster delete "$CLUSTER_NAME" || true rm -rf "$KUBECONFIG" - - # Stop the challenger server - kill $KMS_PID } trap cleanup EXIT -# Create a cluster -k3d cluster create "$CLUSTER_NAME" --image rancher/k3s:v1.26.1-k3s1 +# Create a cluster and bind ports 80 and 443 on the host +# This will allow us to access challenger server on 10.0.2.2 which is the IP +# on which qemu "sees" the host. +k3d cluster create "$CLUSTER_NAME" -p '80:80@server:0' -p '443:443@server:0' --image "$K3S_IMAGE" k3d kubeconfig get "$CLUSTER_NAME" > "$KUBECONFIG" +# Build the docker image +IMG=controller:latest make docker-build + +# Import the image to the cluster +k3d image import -c "$CLUSTER_NAME" controller:latest + # Install cert manager kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml +kubectl wait --for=condition=Available deployment --timeout=2m -n cert-manager --all -# Install the CRDs -kubectl apply -k "$SCRIPT_DIR/../config/crd/" +# Replace the CLUSTER_IP in the kustomize resource +# Only needed for debugging so that we can access the server from the host +# (the 10.0.2.2 IP address is only useful from within qemu) +export CLUSTER_IP=$(docker inspect "k3d-${CLUSTER_NAME}-server-0" | jq -r '.[0].NetworkSettings.Networks[].IPAddress') +envsubst \ + < "$SCRIPT_DIR/../tests/assets/challenger-server-ingress.template.yaml" \ + > "$SCRIPT_DIR/../tests/assets/challenger-server-ingress.yaml" -# Start the challenger server locally -CHALLENGER_PORT=$(getFreePort) -METRICS_PORT=$(getFreePort) -HEALTH_PROBE_PORT=$(getFreePort) -go run "${SCRIPT_DIR}/../" \ - --challenger-bind-address "0.0.0.0:${CHALLENGER_PORT}" \ - --metrics-bind-address "0.0.0.0:${METRICS_PORT}" \ - --health-probe-bind-address "0.0.0.0:${HEALTH_PROBE_PORT}" \ - --namespace default > /dev/null 2>&1 & -export KMS_PID=$! +# Install the challenger server kustomization +kubectl apply -k "$SCRIPT_DIR/../tests/assets/" # 10.0.2.2 is where the vm sees the host # https://stackoverflow.com/a/6752280 -export KMS_ADDRESS="10.0.2.2:${CHALLENGER_PORT}" +export KMS_ADDRESS="10.0.2.2.challenger.sslip.io" PATH=$PATH:$GOPATH/bin ginkgo --nodes $GINKGO_NODES --fail-fast -r ./tests/ diff --git a/tests/assets/challenger-patch.yaml b/tests/assets/challenger-patch.yaml new file mode 100644 index 0000000..0ffb828 --- /dev/null +++ b/tests/assets/challenger-patch.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + # Don't try to pull the image we built locally + imagePullPolicy: IfNotPresent + args: + - "--health-probe-bind-address" + - ":8081" + - "--metrics-bind-address" + - "127.0.0.1:8080" + - "--namespace" + - "default" + - "--leader-elect" + +--- +apiVersion: v1 +kind: Service +metadata: + name: kcrypt-escrow-server + namespace: system +spec: + type: ClusterIP + selector: + control-plane: controller-manager + ports: + - name: wss + port: 8082 + protocol: TCP + targetPort: 8082 diff --git a/tests/assets/challenger-server-ingress.template.yaml b/tests/assets/challenger-server-ingress.template.yaml new file mode 100644 index 0000000..1571871 --- /dev/null +++ b/tests/assets/challenger-server-ingress.template.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: challenger-server + annotations: + cert-manager.io/cluster-issuer: "selfsigned" + kubernetes.io/ingress.class: "traefik" +spec: + tls: + - hosts: + - 10.0.2.2.challenger.sslip.io + - ${CLUSTER_IP}.challenger.sslip.io + secretName: kms-tls + rules: + - host: 10.0.2.2.challenger.sslip.io + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: kcrypt-controller-kcrypt-escrow-server + port: + number: 8082 + - host: ${CLUSTER_IP}.challenger.sslip.io + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: kcrypt-controller-kcrypt-escrow-server + port: + number: 8082 diff --git a/tests/assets/cluster-issuer.yaml b/tests/assets/cluster-issuer.yaml new file mode 100644 index 0000000..4d477d6 --- /dev/null +++ b/tests/assets/cluster-issuer.yaml @@ -0,0 +1,8 @@ +--- +# Self-signed issuer +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned +spec: + selfSigned: {} diff --git a/tests/assets/kustomization.yaml b/tests/assets/kustomization.yaml new file mode 100644 index 0000000..e481c73 --- /dev/null +++ b/tests/assets/kustomization.yaml @@ -0,0 +1,16 @@ +# Adds namespace to all resources. +namespace: default + +bases: +- ../../config/default + +resources: +- challenger-server-ingress.yaml +- cluster-issuer.yaml + +patchesStrategicMerge: +# Fix labels and selectors to make challenger server accessible +- challenger-patch.yaml +# TODO: Implement 2 ingress resources, one for http and one for https (with cert-manager cert) +# or maybe it can be one to server both. The cert should be valid for 10.0.2.2.sslip.io +# which is how the qemu vm will access the server. diff --git a/tests/encryption_test.go b/tests/encryption_test.go index 1759279..c2ee2b1 100644 --- a/tests/encryption_test.go +++ b/tests/encryption_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path" "strconv" + "strings" "syscall" . "github.com/onsi/ginkgo/v2" @@ -27,10 +28,15 @@ var _ = Describe("local encrypted passphrase", func() { }) JustBeforeEach(func() { - out, err := vm.Sudo(fmt.Sprintf(`cat << EOF > config.yaml -%s -`, config)) - Expect(err).ToNot(HaveOccurred(), out) + configFile, err := os.CreateTemp("", "") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(configFile.Name()) + + err = os.WriteFile(configFile.Name(), []byte(config), 0744) + Expect(err).ToNot(HaveOccurred()) + + err = vm.Scp(configFile.Name(), "config.yaml", "0744") + Expect(err).ToNot(HaveOccurred()) installationOutput, err = vm.Sudo("set -o pipefail && kairos-agent manual-install --device auto config.yaml 2>&1 | tee manual-install.txt") Expect(err).ToNot(HaveOccurred(), installationOutput) @@ -68,7 +74,7 @@ hostname: metal-{{ trunc 4 .MachineID }} users: - name: kairos passwd: kairos -EOF` +` }) It("boots and has an encrypted partition", func() { @@ -92,14 +98,14 @@ EOF` apiVersion: keyserver.kairos.io/v1alpha1 kind: SealedVolume metadata: - name: %[1]s - namespace: default + name: "%[1]s" + namespace: default spec: TPMHash: "%[1]s" partitions: - label: COS_PERSISTENT quarantined: false -`, tpmHash)) +`, strings.TrimSpace(tpmHash))) config = fmt.Sprintf(`#cloud-config @@ -121,8 +127,7 @@ kcrypt: nv_index: "" c_index: "" tpm_device: "" - -EOF`, os.Getenv("KMS_ADDRESS")) +`, os.Getenv("KMS_ADDRESS")) }) AfterEach(func() { @@ -145,7 +150,7 @@ EOF`, os.Getenv("KMS_ADDRESS")) ) secretOut, err := cmd.CombinedOutput() - Expect(err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred(), secretOut) Expect(string(secretOut)).To(MatchRegexp("tpm")) }) }) @@ -207,7 +212,7 @@ kcrypt: c_index: "" tpm_device: "" -EOF`, os.Getenv("KMS_ADDRESS")) +`, os.Getenv("KMS_ADDRESS")) }) AfterEach(func() { @@ -229,6 +234,33 @@ EOF`, os.Getenv("KMS_ADDRESS")) Expect(out).To(MatchRegexp("/dev/mapper.*LABEL=\"COS_PERSISTENT\""), out) }) }) + + When("the key management server is listening on https", func() { + BeforeEach(func() { + // TODO: + // - Create and ExternalNames service that points to 10.0.2.2.sslip.io (the server) + // - Create an ingress for the above service with a certificate generated + // by cert-manager + + // Create a service that points to the server running j + // https://github.com/traefik/traefik/issues/1816#issuecomment-322543677 + }) + When("the certificate is pinned on the configuration", func() { + It("successfully talks to the server", func() { + // TODO: Maybe do something simpler than installation to keep things fast? + // Something that proves we talked to the server. + // Cert should be valid for a magic domain (e.g. sslip.io). We can use + // cert-manager to issue one. + }) + }) + When("the certificate signed by a well known CA (system certs)", func() { + It("successfully talks to the server", func() { + // TODO: How do we get a properly signed cert? Maybe do that once, + // and put the cert is the assets directory? + // Is it possible to have a signed cert without a proper domain? + }) + }) + }) }) func printInstallationOutput(message string, callerSkip ...int) {