Implement a test for discoverable KMS

Signed-off-by: Dimitris Karakasilis <dimitris@karakasilis.me>

.github/workflows/e2e-tests.yml

@@ -68,6 +68,7 @@ jobs:
           - label: "remote-static"
           - label: "remote-https-pinned"
           - label: "remote-https-bad-cert"
+          - label: "discoverable-kms"
     steps:
       - name: Checkout code
         uses: actions/checkout@v3

mdns-notes.md

@@ -90,6 +90,7 @@ install:
 # Kcrypt configuration block
 kcrypt:
   challenger:
+    mdns: true
     challenger_server: "http://mychallenger.local"
 ```
 

tests/assets/challenger-server-ingress.template.yaml

@@ -11,6 +11,7 @@ spec:
   - hosts:
     - 10.0.2.2.challenger.sslip.io
     - ${CLUSTER_IP}.challenger.sslip.io
+    - discoverable-kms.local
     secretName: kms-tls
   rules:
     - host: 10.0.2.2.challenger.sslip.io
@@ -33,3 +34,13 @@ spec:
                 name: kcrypt-controller-kcrypt-escrow-server
                 port:
                   number: 8082
+    - host: discoverable-kms.local
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: kcrypt-controller-kcrypt-escrow-server
+                port:
+                  number: 8082

tests/encryption_test.go

@@ -19,13 +19,16 @@ import (
 
 var installationOutput string
 var vm VM
+var mdnsVM VM
 
 var _ = Describe("kcrypt encryption", func() {
 	var config string
+	var vmOpts VMOptions
 
 	BeforeEach(func() {
+		vmOpts = DefaultVMOptions()
 		RegisterFailHandler(printInstallationOutput)
-		_, vm = startVM()
+		_, vm = startVM(vmOpts)
 		fmt.Printf("\nvm.StateDir = %+v\n", vm.StateDir)
 
 		vm.EventuallyConnects(1200)
@@ -63,14 +66,59 @@ var _ = Describe("kcrypt encryption", func() {
 		Expect(err).ToNot(HaveOccurred())
 	})
 
-	When("discovering KMS with mdns", func() {
-		// TODO: Run the simple-mdns-server (https://github.com/kairos-io/simple-mdns-server/)
-		// inside the to-be-installed VM, advertising the KMS as running on 10.0.2.2.
-		// This is a "hack" to avoid setting up 2 VMs just to have the mdns and the
-		// mdns client on the same network. Since our mdns server is just a go binary
-		// and since it can advertise any IP address we want (no necessarily its own),
-		// we will run it inside the VM. It should be enough for the the kcrypt-challenger
-		// cli to get an mdns response.
+	When("discovering KMS with mdns", Label("discoverable-kms"), func() {
+		var tpmHash string
+		var mdnsHostname string
+
+		BeforeEach(func() {
+			By("creating the secret in kubernetes")
+			tpmHash = createTPMPassphraseSecret(vm)
+
+			mdnsHostname = "discoverable-kms.local"
+
+			By("deploying simple-mdns-server vm")
+			mdnsVM = deploySimpleMDNSServer(mdnsHostname)
+
+			config = fmt.Sprintf(`#cloud-config
+
+hostname: metal-{{ trunc 4 .MachineID }}
+users:
+- name: kairos
+  passwd: kairos
+
+install:
+  encrypted_partitions:
+  - COS_PERSISTENT
+  grub_options:
+    extra_cmdline: "rd.neednet=1"
+  reboot: false # we will reboot manually
+
+kcrypt:
+  challenger:
+	  mdns: true
+    challenger_server: "http://%[1]s"
+`, mdnsHostname)
+		})
+
+		AfterEach(func() {
+			cmd := exec.Command("kubectl", "delete", "sealedvolume", tpmHash)
+			out, err := cmd.CombinedOutput()
+			Expect(err).ToNot(HaveOccurred(), out)
+
+			err = mdnsVM.Destroy(func(vm VM) {})
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("discovers the KMS using mdns", func() {
+			By("rebooting")
+			vm.Reboot()
+			By("checking that we can connect after installation")
+			vm.EventuallyConnects(1200)
+			By("checking if we got an encrypted partition")
+			out, err := vm.Sudo("blkid")
+			Expect(err).ToNot(HaveOccurred(), out)
+			Expect(out).To(MatchRegexp("TYPE=\"crypto_LUKS\" PARTLABEL=\"persistent\""), out)
+		})
 	})
 
 	// https://kairos.io/docs/advanced/partition_encryption/#offline-mode
@@ -102,25 +150,9 @@ users:
 	//https://kairos.io/docs/advanced/partition_encryption/#online-mode
 	When("using a remote key management server (automated passphrase generation)", Label("remote-auto"), func() {
 		var tpmHash string
-		var err error
 
 		BeforeEach(func() {
-			tpmHash, err = vm.Sudo("/system/discovery/kcrypt-discovery-challenger")
-			Expect(err).ToNot(HaveOccurred(), tpmHash)
-
-			kubectlApplyYaml(fmt.Sprintf(`---
-apiVersion: keyserver.kairos.io/v1alpha1
-kind: SealedVolume
-metadata:
-  name: "%[1]s"
-  namespace: default
-spec:
-  TPMHash: "%[1]s"
-  partitions:
-    - label: COS_PERSISTENT
-  quarantined: false
-`, strings.TrimSpace(tpmHash)))
-
+			tpmHash = createTPMPassphraseSecret(vm)
 			config = fmt.Sprintf(`#cloud-config
 
 hostname: metal-{{ trunc 4 .MachineID }}
@@ -223,10 +255,6 @@ install:
 kcrypt:
   challenger:
     challenger_server: "http://%s"
-    nv_index: ""
-    c_index: ""
-    tpm_device: ""
-
 `, os.Getenv("KMS_ADDRESS"))
 		})
 
@@ -253,24 +281,15 @@ kcrypt:
 
 	When("the key management server is listening on https", func() {
 		var tpmHash string
-		var err error
 
 		BeforeEach(func() {
-			tpmHash, err = vm.Sudo("/system/discovery/kcrypt-discovery-challenger")
-			Expect(err).ToNot(HaveOccurred(), tpmHash)
+			tpmHash = createTPMPassphraseSecret(vm)
+		})
 
-			kubectlApplyYaml(fmt.Sprintf(`---
-apiVersion: keyserver.kairos.io/v1alpha1
-kind: SealedVolume
-metadata:
-  name: "%[1]s"
-  namespace: default
-spec:
-  TPMHash: "%[1]s"
-  partitions:
-    - label: COS_PERSISTENT
-  quarantined: false
-`, strings.TrimSpace(tpmHash)))
+		AfterEach(func() {
+			cmd := exec.Command("kubectl", "delete", "sealedvolume", tpmHash)
+			out, err := cmd.CombinedOutput()
+			Expect(err).ToNot(HaveOccurred(), out)
 		})
 
 		When("the certificate is pinned on the configuration", Label("remote-https-pinned"), func() {
@@ -327,9 +346,6 @@ install:
 kcrypt:
   challenger:
     challenger_server: "https://%s"
-    nv_index: ""
-    c_index: ""
-    tpm_device: ""
 `, os.Getenv("KMS_ADDRESS"))
 			})
 
@@ -379,3 +395,51 @@ func createConfigWithCert(server, cert string) client.Config {
 
 	return c
 }
+
+func createTPMPassphraseSecret(vm VM) string {
+	tpmHash, err := vm.Sudo("/system/discovery/kcrypt-discovery-challenger")
+	Expect(err).ToNot(HaveOccurred(), tpmHash)
+
+	kubectlApplyYaml(fmt.Sprintf(`---
+apiVersion: keyserver.kairos.io/v1alpha1
+kind: SealedVolume
+metadata:
+  name: "%[1]s"
+  namespace: default
+spec:
+  TPMHash: "%[1]s"
+  partitions:
+    - label: COS_PERSISTENT
+  quarantined: false
+`, strings.TrimSpace(tpmHash)))
+
+	return tpmHash
+}
+
+// We run the simple-mdns-server (https://github.com/kairos-io/simple-mdns-server/)
+// inside a VM next to the one we test. The server advertises the KMS as running on 10.0.2.2
+// (the host machine). This is a "hack" and is needed because of how the default
+// networking in qemu works. We need to be within the same network and that
+// network is only available withing another VM.
+// https://wiki.qemu.org/Documentation/Networking
+func deploySimpleMDNSServer(hostname string) VM {
+	opts := DefaultVMOptions()
+	opts.Memory = "2000"
+	opts.CPUS = "1"
+	opts.EmulateTPM = false
+	_, vm := startVM(opts)
+	vm.EventuallyConnects(1200)
+
+	out, err := vm.Sudo(`curl -s https://api.github.com/repos/kairos-io/simple-mdns-server/releases/latest | jq -r .assets[].browser_download_url | grep $(uname -m) | xargs curl -L -o sms.tar.gz`)
+	Expect(err).ToNot(HaveOccurred(), string(out))
+
+	out, err = vm.Sudo("tar xvf sms.tar.gz")
+	Expect(err).ToNot(HaveOccurred(), string(out))
+
+	// Start the simple-mdns-server in the background
+	out, err = vm.Sudo(fmt.Sprintf(
+		"/bin/bash -c './simple-mdns-server --port 80 --address 10.0.2.2 --serviceType _kcrypt._tcp --hostName %s &'", hostname))
+	Expect(err).ToNot(HaveOccurred(), string(out))
+
+	return vm
+}

tests/suite_test.go

@@ -25,6 +25,47 @@ func TestE2e(t *testing.T) {
 	RunSpecs(t, "kcrypt-challenger e2e test Suite")
 }
 
+type VMOptions struct {
+	ISO        string
+	User       string
+	Password   string
+	Memory     string
+	CPUS       string
+	RunSpicy   bool
+	UseKVM     bool
+	EmulateTPM bool
+}
+
+func DefaultVMOptions() VMOptions {
+	memory := os.Getenv("MEMORY")
+	if memory == "" {
+		memory = "2096"
+	}
+	cpus := os.Getenv("CPUS")
+	if cpus == "" {
+		cpus = "2"
+	}
+	runSpicy, err := strconv.ParseBool(os.Getenv("MACHINE_SPICY"))
+	Expect(err).ToNot(HaveOccurred())
+
+	useKVM := false
+	if envKVM := os.Getenv("KVM"); envKVM != "" {
+		useKVM, err = strconv.ParseBool(os.Getenv("KVM"))
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	return VMOptions{
+		ISO:        os.Getenv("ISO"),
+		User:       user(),
+		Password:   pass(),
+		Memory:     memory,
+		CPUS:       cpus,
+		RunSpicy:   runSpicy,
+		UseKVM:     useKVM,
+		EmulateTPM: true,
+	}
+}
+
 func user() string {
 	user := os.Getenv("SSH_USER")
 	if user == "" {
@@ -42,8 +83,8 @@ func pass() string {
 	return pass
 }
 
-func startVM() (context.Context, VM) {
-	if os.Getenv("ISO") == "" {
+func startVM(vmOpts VMOptions) (context.Context, VM) {
+	if vmOpts.ISO == "" {
 		fmt.Println("ISO missing")
 		os.Exit(1)
 	}
@@ -53,29 +94,22 @@ func startVM() (context.Context, VM) {
 	stateDir, err := os.MkdirTemp("", "")
 	Expect(err).ToNot(HaveOccurred())
 
-	emulateTPM(stateDir)
+	if vmOpts.EmulateTPM {
+		emulateTPM(stateDir)
+	}
 
 	sshPort, err := getFreePort()
 	Expect(err).ToNot(HaveOccurred())
 
-	memory := os.Getenv("MEMORY")
-	if memory == "" {
-		memory = "2096"
-	}
-	cpus := os.Getenv("CPUS")
-	if cpus == "" {
-		cpus = "2"
-	}
-
 	opts := []types.MachineOption{
 		types.QEMUEngine,
-		types.WithISO(os.Getenv("ISO")),
-		types.WithMemory(memory),
-		types.WithCPU(cpus),
+		types.WithISO(vmOpts.ISO),
+		types.WithMemory(vmOpts.Memory),
+		types.WithCPU(vmOpts.CPUS),
 		types.WithSSHPort(strconv.Itoa(sshPort)),
 		types.WithID(vmName),
-		types.WithSSHUser(user()),
-		types.WithSSHPass(pass()),
+		types.WithSSHUser(vmOpts.User),
+		types.WithSSHPass(vmOpts.Password),
 		types.OnFailure(func(p *process.Process) {
 			defer GinkgoRecover()
 
@@ -109,9 +143,12 @@ func startVM() (context.Context, VM) {
 		types.WithStateDir(stateDir),
 		// Serial output to file: https://superuser.com/a/1412150
 		func(m *types.MachineConfig) error {
+			if vmOpts.EmulateTPM {
+				m.Args = append(m.Args,
+					"-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s/swtpm-sock", path.Join(stateDir, "tpm")),
+					"-tpmdev", "emulator,id=tpm0,chardev=chrtpm", "-device", "tpm-tis,tpmdev=tpm0")
+			}
 			m.Args = append(m.Args,
-				"-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s/swtpm-sock", path.Join(stateDir, "tpm")),
-				"-tpmdev", "emulator,id=tpm0,chardev=chrtpm", "-device", "tpm-tis,tpmdev=tpm0",
 				"-chardev", fmt.Sprintf("stdio,mux=on,id=char0,logfile=%s,signal=off", path.Join(stateDir, "serial.log")),
 				"-serial", "chardev:char0",
 				"-mon", "chardev=char0",
@@ -123,14 +160,14 @@ func startVM() (context.Context, VM) {
 	// Set this to true to debug.
 	// You can connect to it with "spicy" or other tool.
 	var spicePort int
-	if os.Getenv("MACHINE_SPICY") != "" {
+	if vmOpts.RunSpicy {
 		spicePort, err = getFreePort()
 		Expect(err).ToNot(HaveOccurred())
 		fmt.Printf("Spice port = %d\n", spicePort)
 		opts = append(opts, types.WithDisplay(fmt.Sprintf("-spice port=%d,addr=127.0.0.1,disable-ticketing", spicePort)))
 	}
 
-	if os.Getenv("KVM") != "" {
+	if vmOpts.UseKVM {
 		opts = append(opts, func(m *types.MachineConfig) error {
 			m.Args = append(m.Args,
 				"-enable-kvm",
@@ -147,7 +184,7 @@ func startVM() (context.Context, VM) {
 	ctx, err := vm.Start(context.Background())
 	Expect(err).ToNot(HaveOccurred())
 
-	if os.Getenv("MACHINE_SPICY") != "" {
+	if vmOpts.RunSpicy {
 		cmd := exec.Command("spicy",
 			"-h", "127.0.0.1",
 			"-p", strconv.Itoa(spicePort))