diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a0ba735..4408068 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -68,6 +68,7 @@ jobs: - label: "remote-static" - label: "remote-https-pinned" - label: "remote-https-bad-cert" + - label: "discoverable-kms" steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/mdns-notes.md b/mdns-notes.md index de1dc6c..298cbcd 100644 --- a/mdns-notes.md +++ b/mdns-notes.md @@ -90,6 +90,7 @@ install: # Kcrypt configuration block kcrypt: challenger: + mdns: true challenger_server: "http://mychallenger.local" ``` diff --git a/tests/assets/challenger-server-ingress.template.yaml b/tests/assets/challenger-server-ingress.template.yaml index 1571871..18ed92c 100644 --- a/tests/assets/challenger-server-ingress.template.yaml +++ b/tests/assets/challenger-server-ingress.template.yaml @@ -11,6 +11,7 @@ spec: - hosts: - 10.0.2.2.challenger.sslip.io - ${CLUSTER_IP}.challenger.sslip.io + - discoverable-kms.local secretName: kms-tls rules: - host: 10.0.2.2.challenger.sslip.io @@ -33,3 +34,13 @@ spec: name: kcrypt-controller-kcrypt-escrow-server port: number: 8082 + - host: discoverable-kms.local + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: kcrypt-controller-kcrypt-escrow-server + port: + number: 8082 diff --git a/tests/encryption_test.go b/tests/encryption_test.go index eb92fb7..f5dd9b3 100644 --- a/tests/encryption_test.go +++ b/tests/encryption_test.go @@ -19,13 +19,16 @@ import ( var installationOutput string var vm VM +var mdnsVM VM var _ = Describe("kcrypt encryption", func() { var config string + var vmOpts VMOptions BeforeEach(func() { + vmOpts = DefaultVMOptions() RegisterFailHandler(printInstallationOutput) - _, vm = startVM() + _, vm = startVM(vmOpts) fmt.Printf("\nvm.StateDir = %+v\n", vm.StateDir) vm.EventuallyConnects(1200) @@ -63,14 +66,59 @@ var _ = Describe("kcrypt encryption", func() { Expect(err).ToNot(HaveOccurred()) }) - When("discovering KMS with mdns", func() { - // TODO: Run the simple-mdns-server (https://github.com/kairos-io/simple-mdns-server/) - // inside the to-be-installed VM, advertising the KMS as running on 10.0.2.2. - // This is a "hack" to avoid setting up 2 VMs just to have the mdns and the - // mdns client on the same network. Since our mdns server is just a go binary - // and since it can advertise any IP address we want (no necessarily its own), - // we will run it inside the VM. It should be enough for the the kcrypt-challenger - // cli to get an mdns response. + When("discovering KMS with mdns", Label("discoverable-kms"), func() { + var tpmHash string + var mdnsHostname string + + BeforeEach(func() { + By("creating the secret in kubernetes") + tpmHash = createTPMPassphraseSecret(vm) + + mdnsHostname = "discoverable-kms.local" + + By("deploying simple-mdns-server vm") + mdnsVM = deploySimpleMDNSServer(mdnsHostname) + + config = fmt.Sprintf(`#cloud-config + +hostname: metal-{{ trunc 4 .MachineID }} +users: +- name: kairos + passwd: kairos + +install: + encrypted_partitions: + - COS_PERSISTENT + grub_options: + extra_cmdline: "rd.neednet=1" + reboot: false # we will reboot manually + +kcrypt: + challenger: + mdns: true + challenger_server: "http://%[1]s" +`, mdnsHostname) + }) + + AfterEach(func() { + cmd := exec.Command("kubectl", "delete", "sealedvolume", tpmHash) + out, err := cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred(), out) + + err = mdnsVM.Destroy(func(vm VM) {}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("discovers the KMS using mdns", func() { + By("rebooting") + vm.Reboot() + By("checking that we can connect after installation") + vm.EventuallyConnects(1200) + By("checking if we got an encrypted partition") + out, err := vm.Sudo("blkid") + Expect(err).ToNot(HaveOccurred(), out) + Expect(out).To(MatchRegexp("TYPE=\"crypto_LUKS\" PARTLABEL=\"persistent\""), out) + }) }) // https://kairos.io/docs/advanced/partition_encryption/#offline-mode @@ -102,25 +150,9 @@ users: //https://kairos.io/docs/advanced/partition_encryption/#online-mode When("using a remote key management server (automated passphrase generation)", Label("remote-auto"), func() { var tpmHash string - var err error BeforeEach(func() { - tpmHash, err = vm.Sudo("/system/discovery/kcrypt-discovery-challenger") - Expect(err).ToNot(HaveOccurred(), tpmHash) - - kubectlApplyYaml(fmt.Sprintf(`--- -apiVersion: keyserver.kairos.io/v1alpha1 -kind: SealedVolume -metadata: - name: "%[1]s" - namespace: default -spec: - TPMHash: "%[1]s" - partitions: - - label: COS_PERSISTENT - quarantined: false -`, strings.TrimSpace(tpmHash))) - + tpmHash = createTPMPassphraseSecret(vm) config = fmt.Sprintf(`#cloud-config hostname: metal-{{ trunc 4 .MachineID }} @@ -223,10 +255,6 @@ install: kcrypt: challenger: challenger_server: "http://%s" - nv_index: "" - c_index: "" - tpm_device: "" - `, os.Getenv("KMS_ADDRESS")) }) @@ -253,24 +281,15 @@ kcrypt: When("the key management server is listening on https", func() { var tpmHash string - var err error BeforeEach(func() { - tpmHash, err = vm.Sudo("/system/discovery/kcrypt-discovery-challenger") - Expect(err).ToNot(HaveOccurred(), tpmHash) + tpmHash = createTPMPassphraseSecret(vm) + }) - kubectlApplyYaml(fmt.Sprintf(`--- -apiVersion: keyserver.kairos.io/v1alpha1 -kind: SealedVolume -metadata: - name: "%[1]s" - namespace: default -spec: - TPMHash: "%[1]s" - partitions: - - label: COS_PERSISTENT - quarantined: false -`, strings.TrimSpace(tpmHash))) + AfterEach(func() { + cmd := exec.Command("kubectl", "delete", "sealedvolume", tpmHash) + out, err := cmd.CombinedOutput() + Expect(err).ToNot(HaveOccurred(), out) }) When("the certificate is pinned on the configuration", Label("remote-https-pinned"), func() { @@ -327,9 +346,6 @@ install: kcrypt: challenger: challenger_server: "https://%s" - nv_index: "" - c_index: "" - tpm_device: "" `, os.Getenv("KMS_ADDRESS")) }) @@ -379,3 +395,51 @@ func createConfigWithCert(server, cert string) client.Config { return c } + +func createTPMPassphraseSecret(vm VM) string { + tpmHash, err := vm.Sudo("/system/discovery/kcrypt-discovery-challenger") + Expect(err).ToNot(HaveOccurred(), tpmHash) + + kubectlApplyYaml(fmt.Sprintf(`--- +apiVersion: keyserver.kairos.io/v1alpha1 +kind: SealedVolume +metadata: + name: "%[1]s" + namespace: default +spec: + TPMHash: "%[1]s" + partitions: + - label: COS_PERSISTENT + quarantined: false +`, strings.TrimSpace(tpmHash))) + + return tpmHash +} + +// We run the simple-mdns-server (https://github.com/kairos-io/simple-mdns-server/) +// inside a VM next to the one we test. The server advertises the KMS as running on 10.0.2.2 +// (the host machine). This is a "hack" and is needed because of how the default +// networking in qemu works. We need to be within the same network and that +// network is only available withing another VM. +// https://wiki.qemu.org/Documentation/Networking +func deploySimpleMDNSServer(hostname string) VM { + opts := DefaultVMOptions() + opts.Memory = "2000" + opts.CPUS = "1" + opts.EmulateTPM = false + _, vm := startVM(opts) + vm.EventuallyConnects(1200) + + out, err := vm.Sudo(`curl -s https://api.github.com/repos/kairos-io/simple-mdns-server/releases/latest | jq -r .assets[].browser_download_url | grep $(uname -m) | xargs curl -L -o sms.tar.gz`) + Expect(err).ToNot(HaveOccurred(), string(out)) + + out, err = vm.Sudo("tar xvf sms.tar.gz") + Expect(err).ToNot(HaveOccurred(), string(out)) + + // Start the simple-mdns-server in the background + out, err = vm.Sudo(fmt.Sprintf( + "/bin/bash -c './simple-mdns-server --port 80 --address 10.0.2.2 --serviceType _kcrypt._tcp --hostName %s &'", hostname)) + Expect(err).ToNot(HaveOccurred(), string(out)) + + return vm +} diff --git a/tests/suite_test.go b/tests/suite_test.go index be2a716..9513a84 100644 --- a/tests/suite_test.go +++ b/tests/suite_test.go @@ -25,6 +25,47 @@ func TestE2e(t *testing.T) { RunSpecs(t, "kcrypt-challenger e2e test Suite") } +type VMOptions struct { + ISO string + User string + Password string + Memory string + CPUS string + RunSpicy bool + UseKVM bool + EmulateTPM bool +} + +func DefaultVMOptions() VMOptions { + memory := os.Getenv("MEMORY") + if memory == "" { + memory = "2096" + } + cpus := os.Getenv("CPUS") + if cpus == "" { + cpus = "2" + } + runSpicy, err := strconv.ParseBool(os.Getenv("MACHINE_SPICY")) + Expect(err).ToNot(HaveOccurred()) + + useKVM := false + if envKVM := os.Getenv("KVM"); envKVM != "" { + useKVM, err = strconv.ParseBool(os.Getenv("KVM")) + Expect(err).ToNot(HaveOccurred()) + } + + return VMOptions{ + ISO: os.Getenv("ISO"), + User: user(), + Password: pass(), + Memory: memory, + CPUS: cpus, + RunSpicy: runSpicy, + UseKVM: useKVM, + EmulateTPM: true, + } +} + func user() string { user := os.Getenv("SSH_USER") if user == "" { @@ -42,8 +83,8 @@ func pass() string { return pass } -func startVM() (context.Context, VM) { - if os.Getenv("ISO") == "" { +func startVM(vmOpts VMOptions) (context.Context, VM) { + if vmOpts.ISO == "" { fmt.Println("ISO missing") os.Exit(1) } @@ -53,29 +94,22 @@ func startVM() (context.Context, VM) { stateDir, err := os.MkdirTemp("", "") Expect(err).ToNot(HaveOccurred()) - emulateTPM(stateDir) + if vmOpts.EmulateTPM { + emulateTPM(stateDir) + } sshPort, err := getFreePort() Expect(err).ToNot(HaveOccurred()) - memory := os.Getenv("MEMORY") - if memory == "" { - memory = "2096" - } - cpus := os.Getenv("CPUS") - if cpus == "" { - cpus = "2" - } - opts := []types.MachineOption{ types.QEMUEngine, - types.WithISO(os.Getenv("ISO")), - types.WithMemory(memory), - types.WithCPU(cpus), + types.WithISO(vmOpts.ISO), + types.WithMemory(vmOpts.Memory), + types.WithCPU(vmOpts.CPUS), types.WithSSHPort(strconv.Itoa(sshPort)), types.WithID(vmName), - types.WithSSHUser(user()), - types.WithSSHPass(pass()), + types.WithSSHUser(vmOpts.User), + types.WithSSHPass(vmOpts.Password), types.OnFailure(func(p *process.Process) { defer GinkgoRecover() @@ -109,9 +143,12 @@ func startVM() (context.Context, VM) { types.WithStateDir(stateDir), // Serial output to file: https://superuser.com/a/1412150 func(m *types.MachineConfig) error { + if vmOpts.EmulateTPM { + m.Args = append(m.Args, + "-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s/swtpm-sock", path.Join(stateDir, "tpm")), + "-tpmdev", "emulator,id=tpm0,chardev=chrtpm", "-device", "tpm-tis,tpmdev=tpm0") + } m.Args = append(m.Args, - "-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s/swtpm-sock", path.Join(stateDir, "tpm")), - "-tpmdev", "emulator,id=tpm0,chardev=chrtpm", "-device", "tpm-tis,tpmdev=tpm0", "-chardev", fmt.Sprintf("stdio,mux=on,id=char0,logfile=%s,signal=off", path.Join(stateDir, "serial.log")), "-serial", "chardev:char0", "-mon", "chardev=char0", @@ -123,14 +160,14 @@ func startVM() (context.Context, VM) { // Set this to true to debug. // You can connect to it with "spicy" or other tool. var spicePort int - if os.Getenv("MACHINE_SPICY") != "" { + if vmOpts.RunSpicy { spicePort, err = getFreePort() Expect(err).ToNot(HaveOccurred()) fmt.Printf("Spice port = %d\n", spicePort) opts = append(opts, types.WithDisplay(fmt.Sprintf("-spice port=%d,addr=127.0.0.1,disable-ticketing", spicePort))) } - if os.Getenv("KVM") != "" { + if vmOpts.UseKVM { opts = append(opts, func(m *types.MachineConfig) error { m.Args = append(m.Args, "-enable-kvm", @@ -147,7 +184,7 @@ func startVM() (context.Context, VM) { ctx, err := vm.Start(context.Background()) Expect(err).ToNot(HaveOccurred()) - if os.Getenv("MACHINE_SPICY") != "" { + if vmOpts.RunSpicy { cmd := exec.Command("spicy", "-h", "127.0.0.1", "-p", strconv.Itoa(spicePort))