diff --git a/src/runtime/Makefile b/src/runtime/Makefile index c6e7ce16d..9f34c3da7 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -1,5 +1,6 @@ # # Copyright (c) 2018-2019 Intel Corporation +# Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -161,6 +162,7 @@ DEFMEMSLOTS := 10 DEFBRIDGES := 1 DEFENABLEANNOTATIONS := [] DEFDISABLEGUESTSECCOMP := true +DEFDISABLEGUESTEMPTYDIR := false #Default experimental features enabled DEFAULTEXPFEATURES := [] @@ -437,6 +439,7 @@ USER_VARS += DEFNETWORKMODEL_ACRN USER_VARS += DEFNETWORKMODEL_CLH USER_VARS += DEFNETWORKMODEL_FC USER_VARS += DEFNETWORKMODEL_QEMU +USER_VARS += DEFDISABLEGUESTEMPTYDIR USER_VARS += DEFDISABLEGUESTSECCOMP USER_VARS += DEFDISABLESELINUX USER_VARS += DEFAULTEXPFEATURES diff --git a/src/runtime/config/configuration-acrn.toml.in b/src/runtime/config/configuration-acrn.toml.in index 2a9736e9e..f0be92ad0 100644 --- a/src/runtime/config/configuration-acrn.toml.in +++ b/src/runtime/config/configuration-acrn.toml.in @@ -1,4 +1,5 @@ # Copyright (c) 2017-2019 Intel Corporation +# Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -219,6 +220,10 @@ disable_selinux=@DEFDISABLESELINUX@ # See: https://pkg.go.dev/github.com/kata-containers/kata-containers/src/runtime/virtcontainers#ContainerType sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index c91260a31..fcf95b2e2 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -1,4 +1,5 @@ # Copyright (c) 2019 Ericsson Eurolab Deutschland GmbH +# Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -309,6 +310,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@ # vfio_mode="@DEFVFIOMODE@" +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/config/configuration-fc.toml.in b/src/runtime/config/configuration-fc.toml.in index e23d32bf8..b6892bd17 100644 --- a/src/runtime/config/configuration-fc.toml.in +++ b/src/runtime/config/configuration-fc.toml.in @@ -1,4 +1,5 @@ # Copyright (c) 2017-2019 Intel Corporation +# Copyright (c) Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -344,6 +345,10 @@ sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ # - When running single containers using a tool like ctr, container sizing information will be available. static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_FC@ +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/config/configuration-qemu.toml.in b/src/runtime/config/configuration-qemu.toml.in index 09486b2fa..422056f7f 100644 --- a/src/runtime/config/configuration-qemu.toml.in +++ b/src/runtime/config/configuration-qemu.toml.in @@ -1,4 +1,5 @@ # Copyright (c) 2017-2019 Intel Corporation +# Copyright (c) 2021 Adobe Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -573,6 +574,10 @@ sandbox_bind_mounts=@DEFBINDMOUNTS@ # vfio_mode="@DEFVFIOMODE@" +# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will +# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest. +disable_guest_empty_dir=@DEFDISABLEGUESTEMPTYDIR@ + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. diff --git a/src/runtime/pkg/containerd-shim-v2/create.go b/src/runtime/pkg/containerd-shim-v2/create.go index 53b049ccf..11609f2bc 100644 --- a/src/runtime/pkg/containerd-shim-v2/create.go +++ b/src/runtime/pkg/containerd-shim-v2/create.go @@ -1,6 +1,7 @@ // Copyright (c) 2014,2015,2016 Docker, Inc. // Copyright (c) 2017 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -57,10 +58,10 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con detach := !r.Terminal ociSpec, bundlePath, err := loadSpec(r) + if err != nil { return nil, err } - containerType, err := oci.ContainerType(*ociSpec) if err != nil { return nil, err @@ -69,16 +70,18 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) rootfs := filepath.Join(r.Bundle, "rootfs") + runtimeConfig, err := loadRuntimeConfig(s, r, ociSpec.Annotations) + if err != nil { + return nil, err + } + switch containerType { case vc.PodSandbox, vc.SingleContainer: if s.sandbox != nil { return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID()) } - s.config, err = loadRuntimeConfig(s, r, ociSpec.Annotations) - if err != nil { - return nil, err - } + s.config = runtimeConfig // create tracer // This is the earliest location we can create the tracer because we must wait @@ -176,7 +179,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con } }() - _, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput) + _, err = katautils.CreateContainer(ctx, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput, runtimeConfig.DisableGuestEmptyDir) if err != nil { return nil, err } diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 0b56fa977..741f32631 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -1,5 +1,6 @@ // Copyright (c) 2018-2022 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -154,6 +155,7 @@ type runtime struct { SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` EnablePprof bool `toml:"enable_pprof"` + DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` } type agent struct { @@ -1173,6 +1175,8 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat } config.SandboxBindMounts = tomlConf.Runtime.SandboxBindMounts + config.DisableGuestEmptyDir = tomlConf.Runtime.DisableGuestEmptyDir + if err := checkConfig(config); err != nil { return "", config, err } diff --git a/src/runtime/pkg/katautils/create.go b/src/runtime/pkg/katautils/create.go index d429558f4..e456d3727 100644 --- a/src/runtime/pkg/katautils/create.go +++ b/src/runtime/pkg/katautils/create.go @@ -1,5 +1,6 @@ // Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -96,12 +97,12 @@ func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeCon // For the given pod ephemeral volume is created only once // backed by tmpfs inside the VM. For successive containers // of the same pod the already existing volume is reused. -func SetEphemeralStorageType(ociSpec specs.Spec) specs.Spec { +func SetEphemeralStorageType(ociSpec specs.Spec, disableGuestEmptyDir bool) specs.Spec { for idx, mnt := range ociSpec.Mounts { if vc.IsEphemeralStorage(mnt.Source) { ociSpec.Mounts[idx].Type = vc.KataEphemeralDevType } - if vc.Isk8sHostEmptyDir(mnt.Source) { + if vc.Isk8sHostEmptyDir(mnt.Source) && !disableGuestEmptyDir { ociSpec.Mounts[idx].Type = vc.KataLocalDevType } } @@ -218,14 +219,14 @@ func checkForFIPS(sandboxConfig *vc.SandboxConfig) error { } // CreateContainer create a container -func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool) (vc.Process, error) { +func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Spec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput bool, disableGuestEmptyDir bool) (vc.Process, error) { var c vc.VCContainer span, ctx := katatrace.Trace(ctx, nil, "CreateContainer", createTracingTags) katatrace.AddTags(span, "container_id", containerID) defer span.End() - ociSpec = SetEphemeralStorageType(ociSpec) + ociSpec = SetEphemeralStorageType(ociSpec, disableGuestEmptyDir) contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput) if err != nil { diff --git a/src/runtime/pkg/katautils/create_test.go b/src/runtime/pkg/katautils/create_test.go index dab665dce..e2488aaa9 100644 --- a/src/runtime/pkg/katautils/create_test.go +++ b/src/runtime/pkg/katautils/create_test.go @@ -1,5 +1,6 @@ // Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -145,7 +146,7 @@ func TestSetEphemeralStorageType(t *testing.T) { ociMounts = append(ociMounts, mount) ociSpec.Mounts = ociMounts - ociSpec = SetEphemeralStorageType(ociSpec) + ociSpec = SetEphemeralStorageType(ociSpec, false) mountType := ociSpec.Mounts[0].Type assert.Equal(mountType, "ephemeral", @@ -367,7 +368,7 @@ func TestCreateContainerContainerConfigFail(t *testing.T) { rootFs := vc.RootFs{Mounted: true} for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput) + _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.Error(err) assert.False(vcmock.IsMockError(err)) assert.True(strings.Contains(err.Error(), containerType)) @@ -395,7 +396,7 @@ func TestCreateContainerFail(t *testing.T) { rootFs := vc.RootFs{Mounted: true} for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput) + _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.Error(err) assert.True(vcmock.IsMockError(err)) } @@ -430,7 +431,7 @@ func TestCreateContainer(t *testing.T) { rootFs := vc.RootFs{Mounted: true} for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput) + _, err = CreateContainer(context.Background(), mockSandbox, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.NoError(err) } } diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index 66172dcba..f72f1147e 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -1,4 +1,5 @@ // Copyright (c) 2017 Intel Corporation +// Copyright (c) 2021 Adobe Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -145,6 +146,9 @@ type RuntimeConfig struct { // Determines if enable pprof EnablePprof bool + + // Determines if Kata creates emptyDir on the guest + DisableGuestEmptyDir bool } // AddKernelParam allows the addition of new kernel parameters to an existing