From 6df165c19d7209571a229e8826e97d33a5aa7a24 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Tue, 22 Sep 2020 15:33:51 -0500 Subject: [PATCH] runtime: add support for SGX Support the `sgx.intel.com/epc` annotation that is defined by the intel k8s plugin. This annotation enables SGX. Hardware-based isolation and memory encryption. For example, use `sgx.intel.com/epc = "64Mi"` to create a container with 1 EPC section with pre-allocated memory. At the time of writing this patch, SGX patches have not landed on the linux kernel project. The following github kernel fork contains all the SGX patches for the host and guest: https://github.com/intel/kvm-sgx fixes #483 Signed-off-by: Julio Montes --- src/runtime/pkg/katautils/config-settings.go.in | 2 ++ src/runtime/pkg/katautils/config.go | 2 ++ src/runtime/pkg/katautils/config_test.go | 2 ++ src/runtime/virtcontainers/clh.go | 9 +++++++++ src/runtime/virtcontainers/hypervisor.go | 4 ++++ src/runtime/virtcontainers/persist.go | 2 ++ src/runtime/virtcontainers/persist/api/config.go | 4 ++++ .../virtcontainers/pkg/annotations/annotations.go | 10 ++++++++++ src/runtime/virtcontainers/pkg/oci/utils.go | 15 +++++++++++++++ src/runtime/virtcontainers/pkg/oci/utils_test.go | 2 ++ 10 files changed, 52 insertions(+) diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index 2aaba8f88e..730b0c8bb8 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -56,6 +56,8 @@ const defaultVhostUserStorePath string = "/var/run/kata-containers/vhost-user/" const defaultRxRateLimiterMaxRate = uint64(0) const defaultTxRateLimiterMaxRate = uint64(0) +var defaultSGXEPCSize = int64(0) + const defaultTemplatePath string = "/run/vc/vm/template" const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index e967b081e9..5a7680920d 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -822,6 +822,7 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { PCIeRootPort: h.PCIeRootPort, DisableVhostNet: true, VirtioFSExtraArgs: h.VirtioFSExtraArgs, + SGXEPCSize: defaultSGXEPCSize, }, nil } @@ -1014,6 +1015,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { DisableImageNvdimm: defaultDisableImageNvdimm, RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate, TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate, + SGXEPCSize: defaultSGXEPCSize, } } diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go index 2fc36dec66..5c3f53c1f4 100644 --- a/src/runtime/pkg/katautils/config_test.go +++ b/src/runtime/pkg/katautils/config_test.go @@ -83,6 +83,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf disableNewNetNs := false sharedFS := "virtio-9p" virtioFSdaemon := path.Join(dir, "virtiofsd") + epcSize := int64(0) configFileOptions := ktu.RuntimeConfigOptions{ Hypervisor: "qemu", @@ -165,6 +166,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf SharedFS: sharedFS, VirtioFSDaemon: virtioFSdaemon, VirtioFSCache: defaultVirtioFSCacheMode, + SGXEPCSize: epcSize, } agentConfig := vc.KataAgentConfig{ diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index 9303e44956..80e5783c4b 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -321,6 +321,15 @@ func (clh *cloudHypervisor) createSandbox(ctx context.Context, id string, networ cache: clh.config.VirtioFSCache, } + if clh.config.SGXEPCSize > 0 { + epcSection := chclient.SgxEpcConfig{ + Size: clh.config.SGXEPCSize, + Prefault: true, + } + + clh.vmconfig.SgxEpc = append(clh.vmconfig.SgxEpc, epcSection) + } + return nil } diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index ccf9434cfd..cf1a53b700 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -415,6 +415,10 @@ type HypervisorConfig struct { // TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level. TxRateLimiterMaxRate uint64 + + // SGXEPCSize specifies the size in bytes for the EPC Section. + // Enable SGX. Hardware-based isolation and memory encryption. + SGXEPCSize int64 } // vcpu mapping from vcpu number to thread number diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 4a1962a12d..74f007d57c 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -247,6 +247,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { VMid: sconfig.HypervisorConfig.VMid, RxRateLimiterMaxRate: sconfig.HypervisorConfig.RxRateLimiterMaxRate, TxRateLimiterMaxRate: sconfig.HypervisorConfig.TxRateLimiterMaxRate, + SGXEPCSize: sconfig.HypervisorConfig.SGXEPCSize, } ss.Config.KataAgentConfig = &persistapi.KataAgentConfig{ @@ -508,6 +509,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { VMid: hconf.VMid, RxRateLimiterMaxRate: hconf.RxRateLimiterMaxRate, TxRateLimiterMaxRate: hconf.TxRateLimiterMaxRate, + SGXEPCSize: hconf.SGXEPCSize, } sconfig.AgentConfig = KataAgentConfig{ diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go index 5bf45ff71a..15092209b9 100644 --- a/src/runtime/virtcontainers/persist/api/config.go +++ b/src/runtime/virtcontainers/persist/api/config.go @@ -185,6 +185,10 @@ type HypervisorConfig struct { // TxRateLimiterMaxRate is used to control network I/O outbound bandwidth on VM level. TxRateLimiterMaxRate uint64 + + // SGXEPCSize specifies the size in bytes for the EPC Section. + // Enable SGX. Hardware-based isolation and memory encryption. + SGXEPCSize int64 } // KataAgentConfig is a structure storing information needed diff --git a/src/runtime/virtcontainers/pkg/annotations/annotations.go b/src/runtime/virtcontainers/pkg/annotations/annotations.go index b9b3bf2fa3..ff9bd13e47 100644 --- a/src/runtime/virtcontainers/pkg/annotations/annotations.go +++ b/src/runtime/virtcontainers/pkg/annotations/annotations.go @@ -270,3 +270,13 @@ const ( // SHA512 is the SHA-512 (64) hash algorithm SHA512 string = "sha512" ) + +// Third-party annotations - annotations defined by other projects or k8s plugins +// but that can change Kata Containers behaviour. + +const ( + // This annotation enables SGX. Hardware-based isolation and memory encryption. + // Supported suffixes are: Ki | Mi | Gi | Ti | Pi | Ei . For example: 4Mi + // For more information about supported suffixes see https://physics.nist.gov/cuu/Units/binary.html + SGXEPC = "sgx.intel.com/epc" +) diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go index 1562c5bab7..08baccb9a9 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils.go +++ b/src/runtime/virtcontainers/pkg/oci/utils.go @@ -19,6 +19,7 @@ import ( crioAnnotations "github.com/cri-o/cri-o/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/resource" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" @@ -442,6 +443,20 @@ func addHypervisorConfigOverrides(ocispec specs.Spec, config *vc.SandboxConfig) config.HypervisorConfig.EntropySource = value } } + if epcSize, ok := ocispec.Annotations[vcAnnotations.SGXEPC]; ok { + quantity, err := resource.ParseQuantity(epcSize) + if err != nil { + return fmt.Errorf("Couldn't parse EPC '%v': %v", err, epcSize) + } + + if quantity.Format != resource.BinarySI { + return fmt.Errorf("Unsupported EPC format '%v': use Ki | Mi | Gi | Ti | Pi | Ei as suffix", epcSize) + } + + size, _ := quantity.AsInt64() + + config.HypervisorConfig.SGXEPCSize = size + } return nil } diff --git a/src/runtime/virtcontainers/pkg/oci/utils_test.go b/src/runtime/virtcontainers/pkg/oci/utils_test.go index 8b011f5bd1..828ecc56d9 100644 --- a/src/runtime/virtcontainers/pkg/oci/utils_test.go +++ b/src/runtime/virtcontainers/pkg/oci/utils_test.go @@ -787,6 +787,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { ocispec.Annotations[vcAnnotations.PCIeRootPort] = "2" ocispec.Annotations[vcAnnotations.EntropySource] = "/dev/urandom" ocispec.Annotations[vcAnnotations.IOMMUPlatform] = "true" + ocispec.Annotations[vcAnnotations.SGXEPC] = "64Mi" // 10Mbit ocispec.Annotations[vcAnnotations.RxRateLimiterMaxRate] = "10000000" ocispec.Annotations[vcAnnotations.TxRateLimiterMaxRate] = "10000000" @@ -823,6 +824,7 @@ func TestAddHypervisorAnnotations(t *testing.T) { assert.Equal(config.HypervisorConfig.PCIeRootPort, uint32(2)) assert.Equal(config.HypervisorConfig.EntropySource, "/dev/urandom") assert.Equal(config.HypervisorConfig.IOMMUPlatform, true) + assert.Equal(config.HypervisorConfig.SGXEPCSize, int64(67108864)) assert.Equal(config.HypervisorConfig.RxRateLimiterMaxRate, uint64(10000000)) assert.Equal(config.HypervisorConfig.TxRateLimiterMaxRate, uint64(10000000))