From f0cb0c7ef72f608fca9999d45c0e334264fa5cac Mon Sep 17 00:00:00 2001 From: fupan Date: Fri, 9 Nov 2018 11:16:17 +0800 Subject: [PATCH] cli: refactor to align with katautils package refactor the cli codes which can be shared with shimv2. Signed-off-by: fupan Signed-off-by: Eric Ernst --- Makefile | 3 - cli/create.go | 222 +------------- cli/create_test.go | 340 ++------------------- cli/delete.go | 13 +- cli/events.go | 3 +- cli/exec.go | 3 +- cli/kata-check.go | 4 +- cli/kata-check_test.go | 3 +- cli/kata-env.go | 5 +- cli/kata-env_test.go | 2 +- cli/kill.go | 3 +- cli/list.go | 3 +- cli/main.go | 15 +- cli/main_test.go | 34 +-- cli/network.go | 136 --------- cli/network_test.go | 136 --------- cli/oci.go | 106 +------ cli/oci_test.go | 103 ------- cli/pause.go | 3 +- cli/ps.go | 3 +- cli/run.go | 3 +- cli/spec.go | 3 +- cli/start.go | 7 +- cli/start_test.go | 3 +- cli/state.go | 3 +- cli/update.go | 3 +- cli/utils.go | 56 +--- cli/utils_test.go | 27 +- cli/version.go | 3 +- pkg/katautils/config-settings.go | 3 +- pkg/katautils/config.go | 56 ++-- pkg/katautils/config_test.go | 10 +- pkg/katautils/create.go | 248 +++++++++++++++ pkg/katautils/create_test.go | 455 ++++++++++++++++++++++++++++ {cli => pkg/katautils}/hook.go | 18 +- {cli => pkg/katautils}/hook_test.go | 27 +- pkg/katautils/logger.go | 2 +- pkg/katautils/logger_test.go | 2 +- pkg/katautils/network.go | 181 +++++++++++ pkg/katautils/network_test.go | 150 +++++++++ pkg/katautils/oci.go | 92 ++++++ pkg/katautils/oci_test.go | 135 +++++++++ {cli => pkg/katautils}/tracing.go | 21 +- pkg/katautils/utils.go | 61 +++- pkg/katautils/utils_test.go | 171 ++++++++++- 45 files changed, 1677 insertions(+), 1203 deletions(-) create mode 100644 pkg/katautils/create.go create mode 100644 pkg/katautils/create_test.go rename {cli => pkg/katautils}/hook.go (84%) rename {cli => pkg/katautils}/hook_test.go (85%) create mode 100644 pkg/katautils/network.go create mode 100644 pkg/katautils/network_test.go create mode 100644 pkg/katautils/oci.go create mode 100644 pkg/katautils/oci_test.go rename {cli => pkg/katautils}/tracing.go (77%) diff --git a/Makefile b/Makefile index 35bb58c02..64693d493 100644 --- a/Makefile +++ b/Makefile @@ -288,9 +288,6 @@ const project = "$(PROJECT_NAME)" // prefix used to denote non-standard CLI commands and options. const projectPrefix = "$(PROJECT_TYPE)" -// systemdUnitName is the systemd(1) target used to launch the agent. -const systemdUnitName = "$(PROJECT_TAG).target" - // original URL for this project const projectURL = "$(PROJECT_URL)" diff --git a/cli/create.go b/cli/create.go index 3cd868b02..5f719299e 100644 --- a/cli/create.go +++ b/cli/create.go @@ -11,10 +11,9 @@ import ( "errors" "fmt" "os" - "strings" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" - vf "github.com/kata-containers/runtime/virtcontainers/factory" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/urfave/cli" ) @@ -87,44 +86,11 @@ var createCLICommand = cli.Command{ }, } -// Use a variable to allow tests to modify its value -var getKernelParamsFunc = getKernelParams - -func handleFactory(ctx context.Context, runtimeConfig oci.RuntimeConfig) { - if !runtimeConfig.FactoryConfig.Template { - return - } - - factoryConfig := vf.Config{ - Template: true, - VMConfig: vc.VMConfig{ - HypervisorType: runtimeConfig.HypervisorType, - HypervisorConfig: runtimeConfig.HypervisorConfig, - AgentType: runtimeConfig.AgentType, - AgentConfig: runtimeConfig.AgentConfig, - }, - } - - kataLog.WithField("factory", factoryConfig).Info("load vm factory") - - f, err := vf.NewFactory(ctx, factoryConfig, true) - if err != nil { - kataLog.WithError(err).Warn("load vm factory failed, about to create new one") - f, err = vf.NewFactory(ctx, factoryConfig, false) - if err != nil { - kataLog.WithError(err).Warn("create vm factory failed") - return - } - } - - vci.SetFactory(ctx, f) -} - func create(ctx context.Context, containerID, bundlePath, console, pidFilePath string, detach, systemdCgroup bool, runtimeConfig oci.RuntimeConfig) error { var err error - span, ctx := trace(ctx, "create") + span, ctx := katautils.Trace(ctx, "create") defer span.Finish() kataLog = kataLog.WithField("container", containerID) @@ -157,19 +123,19 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s return err } - handleFactory(ctx, runtimeConfig) + katautils.HandleFactory(ctx, vci, &runtimeConfig) disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) var process vc.Process switch containerType { case vc.PodSandbox: - process, err = createSandbox(ctx, ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput, systemdCgroup) + _, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput, systemdCgroup, false) if err != nil { return err } case vc.PodContainer: - process, err = createContainer(ctx, ociSpec, containerID, bundlePath, console, disableOutput) + process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, containerID, bundlePath, console, disableOutput, false) if err != nil { return err } @@ -181,184 +147,8 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s return createPIDFile(ctx, pidFilePath, process.Pid) } -var systemdKernelParam = []vc.Param{ - { - Key: "init", - Value: "/usr/lib/systemd/systemd", - }, - { - Key: "systemd.unit", - Value: systemdUnitName, - }, - { - Key: "systemd.mask", - Value: "systemd-networkd.service", - }, - { - Key: "systemd.mask", - Value: "systemd-networkd.socket", - }, -} - -func getKernelParams(needSystemd bool) []vc.Param { - p := []vc.Param{} - - if needSystemd { - p = append(p, systemdKernelParam...) - } - - return p -} - -func needSystemd(config vc.HypervisorConfig) bool { - return config.ImagePath != "" -} - -// setKernelParams adds the user-specified kernel parameters (from the -// configuration file) to the defaults so that the former take priority. -func setKernelParams(containerID string, runtimeConfig *oci.RuntimeConfig) error { - defaultKernelParams := getKernelParamsFunc(needSystemd(runtimeConfig.HypervisorConfig)) - - if runtimeConfig.HypervisorConfig.Debug { - strParams := vc.SerializeParams(defaultKernelParams, "=") - formatted := strings.Join(strParams, " ") - - kataLog.WithField("default-kernel-parameters", formatted).Debug() - } - - // retrieve the parameters specified in the config file - userKernelParams := runtimeConfig.HypervisorConfig.KernelParams - - // reset - runtimeConfig.HypervisorConfig.KernelParams = []vc.Param{} - - // first, add default values - for _, p := range defaultKernelParams { - if err := (runtimeConfig).AddKernelParam(p); err != nil { - return err - } - } - - // now re-add the user-specified values so that they take priority. - for _, p := range userKernelParams { - if err := (runtimeConfig).AddKernelParam(p); err != nil { - return err - } - } - - return nil -} - -func createSandbox(ctx context.Context, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig, - containerID, bundlePath, console string, disableOutput, systemdCgroup bool) (vc.Process, error) { - span, ctx := trace(ctx, "createSandbox") - defer span.Finish() - - err := setKernelParams(containerID, &runtimeConfig) - if err != nil { - return vc.Process{}, err - } - - sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup) - if err != nil { - return vc.Process{}, err - } - - // Important to create the network namespace before the sandbox is - // created, because it is not responsible for the creation of the - // netns if it does not exist. - if err := setupNetworkNamespace(&sandboxConfig.NetworkConfig); err != nil { - return vc.Process{}, err - } - - // Run pre-start OCI hooks. - err = enterNetNS(sandboxConfig.NetworkConfig.NetNSPath, func() error { - return preStartHooks(ctx, ociSpec, containerID, bundlePath) - }) - if err != nil { - return vc.Process{}, err - } - - sandbox, err := vci.CreateSandbox(ctx, sandboxConfig) - if err != nil { - return vc.Process{}, err - } - - sid := sandbox.ID() - kataLog = kataLog.WithField("sandbox", sid) - setExternalLoggers(ctx, kataLog) - span.SetTag("sandbox", sid) - - containers := sandbox.GetAllContainers() - if len(containers) != 1 { - return vc.Process{}, fmt.Errorf("BUG: Container list from sandbox is wrong, expecting only one container, found %d containers", len(containers)) - } - - if err := addContainerIDMapping(ctx, containerID, sandbox.ID()); err != nil { - return vc.Process{}, err - } - - return containers[0].Process(), nil -} - -// setEphemeralStorageType sets the mount type to 'ephemeral' -// if the mount source path is provisioned by k8s for ephemeral storage. -// For the given pod ephemeral volume is created only once -// backed by tmpfs inside the VM. For successive containers -// of the same pod the already existing volume is reused. -func setEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec { - for idx, mnt := range ociSpec.Mounts { - if IsEphemeralStorage(mnt.Source) { - ociSpec.Mounts[idx].Type = "ephemeral" - } - } - return ociSpec -} - -func createContainer(ctx context.Context, ociSpec oci.CompatOCISpec, containerID, bundlePath, - console string, disableOutput bool) (vc.Process, error) { - - span, ctx := trace(ctx, "createContainer") - defer span.Finish() - - ociSpec = setEphemeralStorageType(ociSpec) - - contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput) - if err != nil { - return vc.Process{}, err - } - - sandboxID, err := ociSpec.SandboxID() - if err != nil { - return vc.Process{}, err - } - - kataLog = kataLog.WithField("sandbox", sandboxID) - setExternalLoggers(ctx, kataLog) - span.SetTag("sandbox", sandboxID) - - s, c, err := vci.CreateContainer(ctx, sandboxID, contConfig) - if err != nil { - return vc.Process{}, err - } - - // Run pre-start OCI hooks. - err = enterNetNS(s.GetNetNs(), func() error { - return preStartHooks(ctx, ociSpec, containerID, bundlePath) - }) - if err != nil { - return vc.Process{}, err - } - - if err := addContainerIDMapping(ctx, containerID, sandboxID); err != nil { - return vc.Process{}, err - } - - return c.Process(), nil -} - func createPIDFile(ctx context.Context, pidFilePath string, pid int) error { - span, _ := trace(ctx, "createPIDFile") + span, _ := katautils.Trace(ctx, "createPIDFile") defer span.Finish() if pidFilePath == "" { diff --git a/cli/create_test.go b/cli/create_test.go index 02cd95ddf..da0705c18 100644 --- a/cli/create_test.go +++ b/cli/create_test.go @@ -7,16 +7,15 @@ package main import ( "context" - "errors" "flag" "fmt" "io/ioutil" "os" "path/filepath" "regexp" - "strings" "testing" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" @@ -29,34 +28,14 @@ const ( testPID = 100 testConsole = "/dev/pts/999" testContainerTypeAnnotation = "io.kubernetes.cri-o.ContainerType" - testSandboxIDAnnotation = "io.kubernetes.cri-o.SandboxID" testContainerTypeSandbox = "sandbox" testContainerTypeContainer = "container" ) -var testStrPID = fmt.Sprintf("%d", testPID) - -// return the value of the *last* param with the specified key -func findLastParam(key string, params []vc.Param) (string, error) { - if key == "" { - return "", errors.New("ERROR: need non-nil key") - } - - l := len(params) - if l == 0 { - return "", errors.New("ERROR: no params") - } - - for i := l - 1; i >= 0; i-- { - p := params[i] - - if key == p.Key { - return p.Value, nil - } - } - - return "", fmt.Errorf("no param called %q found", name) -} +var ( + testStrPID = fmt.Sprintf("%d", testPID) + ctrsMapTreePath = "/var/run/kata-containers/containers-mapping" +) func TestCreatePIDFileSuccessful(t *testing.T) { pidDirPath, err := ioutil.TempDir(testDir, "pid-path-") @@ -230,6 +209,7 @@ func TestCreateInvalidArgs(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) defer func() { testingImpl.CreateSandboxFunc = nil @@ -280,6 +260,7 @@ func TestCreateInvalidConfigJSON(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) tmpdir, err := ioutil.TempDir("", "") assert.NoError(err) @@ -296,7 +277,7 @@ func TestCreateInvalidConfigJSON(t *testing.T) { pidFilePath := filepath.Join(tmpdir, "pidfile.txt") ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) + assert.True(katautils.FileExists(ociConfigFile)) f, err := os.OpenFile(ociConfigFile, os.O_APPEND|os.O_WRONLY, testFileMode) assert.NoError(err) @@ -321,6 +302,7 @@ func TestCreateInvalidContainerType(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) tmpdir, err := ioutil.TempDir("", "") assert.NoError(err) @@ -337,7 +319,7 @@ func TestCreateInvalidContainerType(t *testing.T) { pidFilePath := filepath.Join(tmpdir, "pidfile.txt") ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) + assert.True(katautils.FileExists(ociConfigFile)) spec, err := readOCIConfigFile(ociConfigFile) assert.NoError(err) @@ -365,6 +347,7 @@ func TestCreateContainerInvalid(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) tmpdir, err := ioutil.TempDir("", "") assert.NoError(err) @@ -381,7 +364,7 @@ func TestCreateContainerInvalid(t *testing.T) { pidFilePath := filepath.Join(tmpdir, "pidfile.txt") ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) + assert.True(katautils.FileExists(ociConfigFile)) spec, err := readOCIConfigFile(ociConfigFile) @@ -421,6 +404,7 @@ func TestCreateProcessCgroupsPathSuccessful(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) { return sandbox, nil @@ -445,7 +429,7 @@ func TestCreateProcessCgroupsPathSuccessful(t *testing.T) { pidFilePath := filepath.Join(tmpdir, "pidfile.txt") ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) + assert.True(katautils.FileExists(ociConfigFile)) spec, err := readOCIConfigFile(ociConfigFile) assert.NoError(err) @@ -523,6 +507,7 @@ func TestCreateCreateCgroupsFilesFail(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) { return sandbox, nil @@ -547,7 +532,7 @@ func TestCreateCreateCgroupsFilesFail(t *testing.T) { pidFilePath := filepath.Join(tmpdir, "pidfile.txt") ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) + assert.True(katautils.FileExists(ociConfigFile)) spec, err := readOCIConfigFile(ociConfigFile) assert.NoError(err) @@ -608,6 +593,7 @@ func TestCreateCreateCreatePidFileFail(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) { return sandbox, nil @@ -633,7 +619,7 @@ func TestCreateCreateCreatePidFileFail(t *testing.T) { pidFilePath := filepath.Join(pidDir, "pidfile.txt") ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) + assert.True(katautils.FileExists(ociConfigFile)) spec, err := readOCIConfigFile(ociConfigFile) assert.NoError(err) @@ -683,6 +669,7 @@ func TestCreate(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) testingImpl.CreateSandboxFunc = func(ctx context.Context, sandboxConfig vc.SandboxConfig) (vc.VCSandbox, error) { return sandbox, nil @@ -707,7 +694,7 @@ func TestCreate(t *testing.T) { pidFilePath := filepath.Join(tmpdir, "pidfile.txt") ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) + assert.True(katautils.FileExists(ociConfigFile)) spec, err := readOCIConfigFile(ociConfigFile) assert.NoError(err) @@ -741,6 +728,7 @@ func TestCreateInvalidKernelParams(t *testing.T) { assert.NoError(err) defer os.RemoveAll(path) ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(ctrsMapTreePath) tmpdir, err := ioutil.TempDir("", "") assert.NoError(err) @@ -757,7 +745,7 @@ func TestCreateInvalidKernelParams(t *testing.T) { pidFilePath := filepath.Join(tmpdir, "pidfile.txt") ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) + assert.True(katautils.FileExists(ociConfigFile)) spec, err := readOCIConfigFile(ociConfigFile) assert.NoError(err) @@ -770,12 +758,12 @@ func TestCreateInvalidKernelParams(t *testing.T) { err = writeOCIConfigFile(spec, ociConfigFile) assert.NoError(err) - savedFunc := getKernelParamsFunc + savedFunc := katautils.GetKernelParamsFunc defer func() { - getKernelParamsFunc = savedFunc + katautils.GetKernelParamsFunc = savedFunc }() - getKernelParamsFunc = func(needSystemd bool) []vc.Param { + katautils.GetKernelParamsFunc = func(needSystemd bool) []vc.Param { return []vc.Param{ { Key: "", @@ -791,281 +779,3 @@ func TestCreateInvalidKernelParams(t *testing.T) { os.RemoveAll(path) } } - -func TestCreateSandboxConfigFail(t *testing.T) { - assert := assert.New(t) - - path, err := ioutil.TempDir("", "containers-mapping") - assert.NoError(err) - defer os.RemoveAll(path) - ctrsMapTreePath = path - - tmpdir, err := ioutil.TempDir("", "") - assert.NoError(err) - defer os.RemoveAll(tmpdir) - - runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) - assert.NoError(err) - - bundlePath := filepath.Join(tmpdir, "bundle") - - err = makeOCIBundle(bundlePath) - assert.NoError(err) - - ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) - - spec, err := readOCIConfigFile(ociConfigFile) - assert.NoError(err) - - quota := int64(0) - limit := int64(0) - - spec.Linux.Resources.Memory = &specs.LinuxMemory{ - Limit: &limit, - } - - spec.Linux.Resources.CPU = &specs.LinuxCPU{ - // specify an invalid value - Quota: "a, - } - - _, err = createSandbox(context.Background(), spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true) - assert.Error(err) -} - -func TestCreateCreateSandboxFail(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledNeedNonRoot) - } - - assert := assert.New(t) - - path, err := ioutil.TempDir("", "containers-mapping") - assert.NoError(err) - defer os.RemoveAll(path) - ctrsMapTreePath = path - - tmpdir, err := ioutil.TempDir("", "") - assert.NoError(err) - defer os.RemoveAll(tmpdir) - - runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) - assert.NoError(err) - - bundlePath := filepath.Join(tmpdir, "bundle") - - err = makeOCIBundle(bundlePath) - assert.NoError(err) - - ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) - - spec, err := readOCIConfigFile(ociConfigFile) - assert.NoError(err) - - _, err = createSandbox(context.Background(), spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true) - assert.Error(err) - assert.True(vcmock.IsMockError(err)) -} - -func TestCreateCreateContainerContainerConfigFail(t *testing.T) { - assert := assert.New(t) - - path, err := ioutil.TempDir("", "containers-mapping") - assert.NoError(err) - defer os.RemoveAll(path) - ctrsMapTreePath = path - - tmpdir, err := ioutil.TempDir("", "") - assert.NoError(err) - defer os.RemoveAll(tmpdir) - - bundlePath := filepath.Join(tmpdir, "bundle") - - err = makeOCIBundle(bundlePath) - assert.NoError(err) - - ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) - - spec, err := readOCIConfigFile(ociConfigFile) - assert.NoError(err) - - // Set invalid container type - containerType := "你好，世界" - spec.Annotations = make(map[string]string) - spec.Annotations[testContainerTypeAnnotation] = containerType - - // rewrite file - err = writeOCIConfigFile(spec, ociConfigFile) - assert.NoError(err) - - for _, disableOutput := range []bool{true, false} { - _, err = createContainer(context.Background(), spec, testContainerID, bundlePath, testConsole, disableOutput) - assert.Error(err) - assert.False(vcmock.IsMockError(err)) - assert.True(strings.Contains(err.Error(), containerType)) - os.RemoveAll(path) - } -} - -func TestCreateCreateContainerFail(t *testing.T) { - assert := assert.New(t) - - path, err := ioutil.TempDir("", "containers-mapping") - assert.NoError(err) - defer os.RemoveAll(path) - ctrsMapTreePath = path - - tmpdir, err := ioutil.TempDir("", "") - assert.NoError(err) - defer os.RemoveAll(tmpdir) - - bundlePath := filepath.Join(tmpdir, "bundle") - - err = makeOCIBundle(bundlePath) - assert.NoError(err) - - ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) - - spec, err := readOCIConfigFile(ociConfigFile) - assert.NoError(err) - - // set expected container type and sandboxID - spec.Annotations = make(map[string]string) - spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer - spec.Annotations[testSandboxIDAnnotation] = testSandboxID - - // rewrite file - err = writeOCIConfigFile(spec, ociConfigFile) - assert.NoError(err) - - for _, disableOutput := range []bool{true, false} { - _, err = createContainer(context.Background(), spec, testContainerID, bundlePath, testConsole, disableOutput) - assert.Error(err) - assert.True(vcmock.IsMockError(err)) - os.RemoveAll(path) - } -} - -func TestSetEphemeralStorageType(t *testing.T) { - assert := assert.New(t) - - ociSpec := oci.CompatOCISpec{} - var ociMounts []specs.Mount - mount := specs.Mount{ - Source: "/var/lib/kubelet/pods/366c3a77-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume", - } - - ociMounts = append(ociMounts, mount) - ociSpec.Mounts = ociMounts - ociSpec = setEphemeralStorageType(ociSpec) - - mountType := ociSpec.Mounts[0].Type - assert.Equal(mountType, "ephemeral", - "Unexpected mount type, got %s expected ephemeral", mountType) -} - -func TestCreateCreateContainer(t *testing.T) { - assert := assert.New(t) - - path, err := ioutil.TempDir("", "containers-mapping") - assert.NoError(err) - defer os.RemoveAll(path) - ctrsMapTreePath = path - - testingImpl.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) { - return &vcmock.Sandbox{}, &vcmock.Container{}, nil - } - - defer func() { - testingImpl.CreateContainerFunc = nil - }() - - tmpdir, err := ioutil.TempDir("", "") - assert.NoError(err) - defer os.RemoveAll(tmpdir) - - bundlePath := filepath.Join(tmpdir, "bundle") - - err = makeOCIBundle(bundlePath) - assert.NoError(err) - - ociConfigFile := filepath.Join(bundlePath, "config.json") - assert.True(fileExists(ociConfigFile)) - - spec, err := readOCIConfigFile(ociConfigFile) - assert.NoError(err) - - // set expected container type and sandboxID - spec.Annotations = make(map[string]string) - spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer - spec.Annotations[testSandboxIDAnnotation] = testSandboxID - - // rewrite file - err = writeOCIConfigFile(spec, ociConfigFile) - assert.NoError(err) - - for _, disableOutput := range []bool{true, false} { - _, err = createContainer(context.Background(), spec, testContainerID, bundlePath, testConsole, disableOutput) - assert.NoError(err) - os.RemoveAll(path) - } -} - -func TestSetKernelParams(t *testing.T) { - assert := assert.New(t) - - config := oci.RuntimeConfig{} - - assert.Empty(config.HypervisorConfig.KernelParams) - - err := setKernelParams(testContainerID, &config) - assert.NoError(err) - - if needSystemd(config.HypervisorConfig) { - assert.NotEmpty(config.HypervisorConfig.KernelParams) - } -} - -func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) { - assert := assert.New(t) - - initName := "init" - initValue := "/sbin/myinit" - - ipName := "ip" - ipValue := "127.0.0.1" - - params := []vc.Param{ - {Key: initName, Value: initValue}, - {Key: ipName, Value: ipValue}, - } - - hypervisorConfig := vc.HypervisorConfig{ - KernelParams: params, - } - - // Config containing user-specified kernel parameters - config := oci.RuntimeConfig{ - HypervisorConfig: hypervisorConfig, - } - - assert.NotEmpty(config.HypervisorConfig.KernelParams) - - err := setKernelParams(testContainerID, &config) - assert.NoError(err) - - kernelParams := config.HypervisorConfig.KernelParams - - init, err := findLastParam(initName, kernelParams) - assert.NoError(err) - assert.Equal(initValue, init) - - ip, err := findLastParam(ipName, kernelParams) - assert.NoError(err) - assert.Equal(ipValue, ip) - -} diff --git a/cli/delete.go b/cli/delete.go index d7104f47d..cb7453902 100644 --- a/cli/delete.go +++ b/cli/delete.go @@ -11,6 +11,7 @@ import ( "fmt" "os" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" vcAnnot "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" @@ -60,7 +61,7 @@ EXAMPLE: } func delete(ctx context.Context, containerID string, force bool) error { - span, ctx := trace(ctx, "delete") + span, ctx := katautils.Trace(ctx, "delete") defer span.Finish() kataLog = kataLog.WithField("container", containerID) @@ -119,15 +120,15 @@ func delete(ctx context.Context, containerID string, force bool) error { } // Run post-stop OCI hooks. - if err := postStopHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey]); err != nil { + if err := katautils.PostStopHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey]); err != nil { return err } - return delContainerIDMapping(ctx, containerID) + return katautils.DelContainerIDMapping(ctx, containerID) } func deleteSandbox(ctx context.Context, sandboxID string) error { - span, _ := trace(ctx, "deleteSandbox") + span, _ := katautils.Trace(ctx, "deleteSandbox") defer span.Finish() status, err := vci.StatusSandbox(ctx, sandboxID) @@ -149,7 +150,7 @@ func deleteSandbox(ctx context.Context, sandboxID string) error { } func deleteContainer(ctx context.Context, sandboxID, containerID string, forceStop bool) error { - span, _ := trace(ctx, "deleteContainer") + span, _ := katautils.Trace(ctx, "deleteContainer") defer span.Finish() if forceStop { @@ -166,7 +167,7 @@ func deleteContainer(ctx context.Context, sandboxID, containerID string, forceSt } func removeCgroupsPath(ctx context.Context, containerID string, cgroupsPathList []string) error { - span, _ := trace(ctx, "removeCgroupsPath") + span, _ := katautils.Trace(ctx, "removeCgroupsPath") defer span.Finish() if len(cgroupsPathList) == 0 { diff --git a/cli/events.go b/cli/events.go index e50ae1296..96acce621 100644 --- a/cli/events.go +++ b/cli/events.go @@ -15,6 +15,7 @@ import ( vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/pkg/katautils" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -141,7 +142,7 @@ information is displayed once every 5 seconds.`, return err } - span, _ := trace(ctx, "events") + span, _ := katautils.Trace(ctx, "events") defer span.Finish() containerID := context.Args().First() diff --git a/cli/exec.go b/cli/exec.go index de559b026..549c29210 100644 --- a/cli/exec.go +++ b/cli/exec.go @@ -14,6 +14,7 @@ import ( "os" "syscall" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -188,7 +189,7 @@ func generateExecParams(context *cli.Context, specProcess *oci.CompatOCIProcess) } func execute(ctx context.Context, context *cli.Context) error { - span, ctx := trace(ctx, "execute") + span, ctx := katautils.Trace(ctx, "execute") defer span.Finish() containerID := context.Args().First() diff --git a/cli/kata-check.go b/cli/kata-check.go index 713a3e567..0dd32995d 100644 --- a/cli/kata-check.go +++ b/cli/kata-check.go @@ -117,7 +117,7 @@ func getCPUFlags(cpuinfo string) string { func haveKernelModule(module string) bool { // First, check to see if the module is already loaded path := filepath.Join(sysModuleDir, module) - if fileExists(path) { + if katautils.FileExists(path) { return true } @@ -288,7 +288,7 @@ var kataCheckCLICommand = cli.Command{ return err } - span, _ := trace(ctx, "kata-check") + span, _ := katautils.Trace(ctx, "kata-check") defer span.Finish() setCPUtype() diff --git a/cli/kata-check_test.go b/cli/kata-check_test.go index 502f280c6..830e6260b 100644 --- a/cli/kata-check_test.go +++ b/cli/kata-check_test.go @@ -15,6 +15,7 @@ import ( "path/filepath" "testing" + "github.com/kata-containers/runtime/pkg/katautils" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/urfave/cli" @@ -59,7 +60,7 @@ func createModules(assert *assert.Assertions, cpuInfoFile string, moduleData []t } err = hostIsVMContainerCapable(details) - if fileExists(cpuInfoFile) { + if katautils.FileExists(cpuInfoFile) { assert.NoError(err) } else { assert.Error(err) diff --git a/cli/kata-env.go b/cli/kata-env.go index 40fbde85f..0770ff6fd 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -14,6 +14,7 @@ import ( runtim "runtime" "github.com/BurntSushi/toml" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" vcUtils "github.com/kata-containers/runtime/virtcontainers/utils" @@ -274,7 +275,7 @@ func getNetmonInfo(config oci.RuntimeConfig) (NetmonInfo, error) { } func getCommandVersion(cmd string) (string, error) { - return runCommand([]string{cmd, "--version"}) + return katautils.RunCommand([]string{cmd, "--version"}) } func getShimInfo(config oci.RuntimeConfig) (ShimInfo, error) { @@ -451,7 +452,7 @@ var kataEnvCLICommand = cli.Command{ return err } - span, _ := trace(ctx, "kata-env") + span, _ := katautils.Trace(ctx, "kata-env") defer span.Finish() return handleSettings(defaultOutputFile, context) diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go index 298010d44..b48740852 100644 --- a/cli/kata-env_test.go +++ b/cli/kata-env_test.go @@ -193,7 +193,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC return "", oci.RuntimeConfig{}, err } - _, config, _, err = katautils.LoadConfiguration(configFile, true, false) + _, config, err = katautils.LoadConfiguration(configFile, true, false) if err != nil { return "", oci.RuntimeConfig{}, err } diff --git a/cli/kill.go b/cli/kill.go index 0f47e3b13..bb5e7a795 100644 --- a/cli/kill.go +++ b/cli/kill.go @@ -12,6 +12,7 @@ import ( "strconv" "syscall" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/sirupsen/logrus" @@ -97,7 +98,7 @@ var signalList = map[string]syscall.Signal{ } func kill(ctx context.Context, containerID, signal string, all bool) error { - span, _ := trace(ctx, "kill") + span, _ := katautils.Trace(ctx, "kill") defer span.Finish() kataLog = kataLog.WithField("container", containerID) diff --git a/cli/list.go b/cli/list.go index 46baffa03..26dc7391b 100644 --- a/cli/list.go +++ b/cli/list.go @@ -19,6 +19,7 @@ import ( "github.com/urfave/cli" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" oci "github.com/kata-containers/runtime/virtcontainers/pkg/oci" ) @@ -114,7 +115,7 @@ To list containers created using a non-default value for "--root": return err } - span, ctx := trace(ctx, "list") + span, ctx := katautils.Trace(ctx, "list") defer span.Finish() s, err := getContainers(ctx, context) diff --git a/cli/main.go b/cli/main.go index 607fd9266..a01a02dc5 100644 --- a/cli/main.go +++ b/cli/main.go @@ -60,9 +60,6 @@ var originalLoggerLevel logrus.Level var debug = false -// if true, enable opentracing support. -var tracing = false - // if true, coredump when an internal error occurs or a fatal signal is received var crashOnError = false @@ -193,7 +190,7 @@ func setupSignalHandler(ctx context.Context) { } dieCb := func() { - stopTracing(ctx) + katautils.StopTracing(ctx) } go func() { @@ -229,7 +226,7 @@ func setExternalLoggers(ctx context.Context, logger *logrus.Entry) { // created. if opentracing.SpanFromContext(ctx) != nil { - span, ctx = trace(ctx, "setExternalLoggers") + span, ctx = katautils.Trace(ctx, "setExternalLoggers") defer span.Finish() } @@ -307,7 +304,7 @@ func beforeSubcommands(c *cli.Context) error { katautils.SetConfigOptions(name, defaultRuntimeConfiguration, defaultSysConfRuntimeConfiguration) - configFile, runtimeConfig, tracing, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreLogging, false) + configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreLogging, false) if err != nil { fatal(err) } @@ -360,7 +357,7 @@ func handleShowConfig(context *cli.Context) { } func setupTracing(context *cli.Context, rootSpanName string) error { - tracer, err := createTracer(name) + tracer, err := katautils.CreateTracer(name) if err != nil { fatal(err) } @@ -397,7 +394,7 @@ func afterSubcommands(c *cli.Context) error { return err } - stopTracing(ctx) + katautils.StopTracing(ctx) return nil } @@ -546,7 +543,7 @@ func main() { ctx := context.Background() dieCb := func() { - stopTracing(ctx) + katautils.StopTracing(ctx) } defer signals.HandlePanic(dieCb) diff --git a/cli/main_test.go b/cli/main_test.go index 161406c03..b9843a5c7 100644 --- a/cli/main_test.go +++ b/cli/main_test.go @@ -33,7 +33,6 @@ import ( ) const ( - testDisabledNeedRoot = "Test disabled as requires root user" testDisabledNeedNonRoot = "Test disabled as requires non-root user" testDirMode = os.FileMode(0750) testFileMode = os.FileMode(0640) @@ -90,7 +89,7 @@ func init() { fmt.Printf("INFO: test directory is %v\n", testDir) fmt.Printf("INFO: ensuring docker is running\n") - output, err := runCommandFull([]string{"docker", "version"}, true) + output, err := katautils.RunCommandFull([]string{"docker", "version"}, true) if err != nil { panic(fmt.Sprintf("ERROR: docker daemon is not installed, not running, or not accessible to current user: %v (error %v)", output, err)) @@ -101,11 +100,11 @@ func init() { fmt.Printf("INFO: ensuring required docker image (%v) is available\n", testDockerImage) // Only hit the network if the image doesn't exist locally - _, err = runCommand([]string{"docker", "inspect", "--type=image", testDockerImage}) + _, err = katautils.RunCommand([]string{"docker", "inspect", "--type=image", testDockerImage}) if err == nil { fmt.Printf("INFO: docker image %v already exists locally\n", testDockerImage) } else { - _, err = runCommand([]string{"docker", "pull", testDockerImage}) + _, err = katautils.RunCommand([]string{"docker", "pull", testDockerImage}) if err != nil { panic(err) } @@ -256,7 +255,7 @@ func createOCIConfig(bundleDir string) error { return errors.New("BUG: Need bundle directory") } - if !fileExists(bundleDir) { + if !katautils.FileExists(bundleDir) { return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir) } @@ -276,13 +275,13 @@ func createOCIConfig(bundleDir string) error { return errors.New("Cannot find command to generate OCI config file") } - _, err := runCommand([]string{configCmd, "spec", "--bundle", bundleDir}) + _, err := katautils.RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) if err != nil { return err } specFile := filepath.Join(bundleDir, specConfig) - if !fileExists(specFile) { + if !katautils.FileExists(specFile) { return fmt.Errorf("generated OCI config file does not exist: %v", specFile) } @@ -297,7 +296,7 @@ func createRootfs(dir string) error { return err } - container, err := runCommand([]string{"docker", "create", testDockerImage}) + container, err := katautils.RunCommand([]string{"docker", "create", testDockerImage}) if err != nil { return err } @@ -328,7 +327,7 @@ func createRootfs(dir string) error { } // Clean up - _, err = runCommand([]string{"docker", "rm", container}) + _, err = katautils.RunCommand([]string{"docker", "rm", container}) if err != nil { return err } @@ -346,7 +345,7 @@ func realMakeOCIBundle(bundleDir string) error { return errors.New("BUG: Need bundle directory") } - if !fileExists(bundleDir) { + if !katautils.FileExists(bundleDir) { return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir) } @@ -396,12 +395,12 @@ func makeOCIBundle(bundleDir string) error { base := filepath.Dir(bundleDir) for _, dir := range []string{from, base} { - if !fileExists(dir) { + if !katautils.FileExists(dir) { return fmt.Errorf("BUG: directory %v should exist", dir) } } - output, err := runCommandFull([]string{"cp", "-a", from, to}, true) + output, err := katautils.RunCommandFull([]string{"cp", "-a", from, to}, true) if err != nil { return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output) } @@ -501,7 +500,7 @@ func TestMakeOCIBundle(t *testing.T) { assert.NoError(err) specFile := filepath.Join(bundleDir, specConfig) - assert.True(fileExists(specFile)) + assert.True(katautils.FileExists(specFile)) } func TestCreateOCIConfig(t *testing.T) { @@ -524,7 +523,7 @@ func TestCreateOCIConfig(t *testing.T) { assert.NoError(err) specFile := filepath.Join(bundleDir, specConfig) - assert.True(fileExists(specFile)) + assert.True(katautils.FileExists(specFile)) } func TestCreateRootfs(t *testing.T) { @@ -535,7 +534,7 @@ func TestCreateRootfs(t *testing.T) { defer os.RemoveAll(tmpdir) rootfsDir := filepath.Join(tmpdir, "rootfs") - assert.False(fileExists(rootfsDir)) + assert.False(katautils.FileExists(rootfsDir)) err = createRootfs(rootfsDir) assert.NoError(err) @@ -543,11 +542,11 @@ func TestCreateRootfs(t *testing.T) { // non-comprehensive list of expected directories expectedDirs := []string{"bin", "dev", "etc", "usr", "var"} - assert.True(fileExists(rootfsDir)) + assert.True(katautils.FileExists(rootfsDir)) for _, dir := range expectedDirs { dirPath := filepath.Join(rootfsDir, dir) - assert.True(fileExists(dirPath)) + assert.True(katautils.FileExists(dirPath)) } } @@ -1122,5 +1121,6 @@ func createTempContainerIDMapping(containerID, sandboxID string) (string, error) return "", err } + katautils.SetCtrsMapTreePath(ctrsMapTreePath) return tmpDir, nil } diff --git a/cli/network.go b/cli/network.go index 57e367c2f..91b171a17 100644 --- a/cli/network.go +++ b/cli/network.go @@ -6,17 +6,11 @@ package main import ( - "bufio" "context" "encoding/json" "fmt" "os" - "path/filepath" - "strings" - "golang.org/x/sys/unix" - - "github.com/containernetworking/plugins/pkg/ns" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/sirupsen/logrus" @@ -233,133 +227,3 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT } return err } - -const procMountInfoFile = "/proc/self/mountinfo" - -// getNetNsFromBindMount returns the network namespace for the bind-mounted path -func getNetNsFromBindMount(nsPath string, procMountFile string) (string, error) { - netNsMountType := "nsfs" - - // Resolve all symlinks in the path as the mountinfo file contains - // resolved paths. - nsPath, err := filepath.EvalSymlinks(nsPath) - if err != nil { - return "", err - } - - f, err := os.Open(procMountFile) - if err != nil { - return "", err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - text := scanner.Text() - - // Scan the mountinfo file to search for the network namespace path - // This file contains mounts in the eg format: - // "711 26 0:3 net:[4026532009] /run/docker/netns/default rw shared:535 - nsfs nsfs rw" - // - // Reference: https://www.kernel.org/doc/Documentation/filesystems/proc.txt - - // We are interested in the first 9 fields of this file, - // to check for the correct mount type. - fields := strings.Split(text, " ") - if len(fields) < 9 { - continue - } - - // We check here if the mount type is a network namespace mount type, namely "nsfs" - mountTypeFieldIdx := 8 - if fields[mountTypeFieldIdx] != netNsMountType { - continue - } - - // This is the mount point/destination for the mount - mntDestIdx := 4 - if fields[mntDestIdx] != nsPath { - continue - } - - // This is the root/source of the mount - return fields[3], nil - } - - return "", nil -} - -// hostNetworkingRequested checks if the network namespace requested is the -// same as the current process. -func hostNetworkingRequested(configNetNs string) (bool, error) { - var evalNS, nsPath, currentNsPath string - var err error - - // Net namespace provided as "/proc/pid/ns/net" or "/proc//task//ns/net" - if strings.HasPrefix(configNetNs, "/proc") && strings.HasSuffix(configNetNs, "/ns/net") { - if _, err := os.Stat(configNetNs); err != nil { - return false, err - } - - // Here we are trying to resolve the path but it fails because - // namespaces links don't really exist. For this reason, the - // call to EvalSymlinks will fail when it will try to stat the - // resolved path found. As we only care about the path, we can - // retrieve it from the PathError structure. - if _, err = filepath.EvalSymlinks(configNetNs); err != nil { - nsPath = err.(*os.PathError).Path - } else { - return false, fmt.Errorf("Net namespace path %s is not a symlink", configNetNs) - } - - _, evalNS = filepath.Split(nsPath) - - } else { - // Bind-mounted path provided - evalNS, _ = getNetNsFromBindMount(configNetNs, procMountInfoFile) - } - - currentNS := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid()) - if _, err = filepath.EvalSymlinks(currentNS); err != nil { - currentNsPath = err.(*os.PathError).Path - } else { - return false, fmt.Errorf("Unexpected: Current network namespace path is not a symlink") - } - - _, evalCurrentNS := filepath.Split(currentNsPath) - - if evalNS == evalCurrentNS { - return true, nil - } - - return false, nil -} - -func setupNetworkNamespace(config *vc.NetworkConfig) error { - if config.DisableNewNetNs { - kataLog.Info("DisableNewNetNs is on, shim and hypervisor are running in the host netns") - return nil - } - - if config.NetNSPath == "" { - n, err := ns.NewNS() - if err != nil { - return err - } - - config.NetNSPath = n.Path() - config.NetNsCreated = true - - return nil - } - - isHostNs, err := hostNetworkingRequested(config.NetNSPath) - if err != nil { - return err - } - if isHostNs { - return fmt.Errorf("Host networking requested, not supported by runtime") - } - - return nil -} diff --git a/cli/network_test.go b/cli/network_test.go index 5a7ede4f6..2853a169d 100644 --- a/cli/network_test.go +++ b/cli/network_test.go @@ -8,16 +8,10 @@ package main import ( "context" "flag" - "fmt" "io/ioutil" "os" - "path/filepath" - "syscall" "testing" - "golang.org/x/sys/unix" - - "github.com/containernetworking/plugins/pkg/ns" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/types" "github.com/stretchr/testify/assert" @@ -93,133 +87,3 @@ func TestNetworkCliFunction(t *testing.T) { f.Close() execCLICommandFunc(assert, updateRoutesCommand, set, false) } - -func TestGetNetNsFromBindMount(t *testing.T) { - assert := assert.New(t) - - tmpdir, err := ioutil.TempDir("", "") - assert.NoError(err) - defer os.RemoveAll(tmpdir) - - mountFile := filepath.Join(tmpdir, "mountInfo") - nsPath := filepath.Join(tmpdir, "ns123") - - // Non-existent namespace path - _, err = getNetNsFromBindMount(nsPath, mountFile) - assert.NotNil(err) - - tmpNSPath := filepath.Join(tmpdir, "testNetNs") - f, err := os.Create(tmpNSPath) - assert.NoError(err) - defer f.Close() - - type testData struct { - contents string - expectedResult string - } - - data := []testData{ - {fmt.Sprintf("711 26 0:3 net:[4026532008] %s rw shared:535 - nsfs nsfs rw", tmpNSPath), "net:[4026532008]"}, - {"711 26 0:3 net:[4026532008] /run/netns/ns123 rw shared:535 - tmpfs tmpfs rw", ""}, - {"a a a a a a a - b c d", ""}, - {"", ""}, - } - - for i, d := range data { - err := ioutil.WriteFile(mountFile, []byte(d.contents), 0640) - assert.NoError(err) - - path, err := getNetNsFromBindMount(tmpNSPath, mountFile) - assert.NoError(err, fmt.Sprintf("got %q, test data: %+v", path, d)) - - assert.Equal(d.expectedResult, path, "Test %d, expected %s, got %s", i, d.expectedResult, path) - } -} - -func TestHostNetworkingRequested(t *testing.T) { - assert := assert.New(t) - - if os.Geteuid() != 0 { - t.Skip(testDisabledNeedRoot) - } - - // Network namespace same as the host - selfNsPath := "/proc/self/ns/net" - isHostNs, err := hostNetworkingRequested(selfNsPath) - assert.NoError(err) - assert.True(isHostNs) - - // Non-existent netns path - nsPath := "/proc/123456789/ns/net" - _, err = hostNetworkingRequested(nsPath) - assert.Error(err) - - // Bind-mounted Netns - tmpdir, err := ioutil.TempDir("", "") - assert.NoError(err) - defer os.RemoveAll(tmpdir) - - // Create a bind mount to the current network namespace. - tmpFile := filepath.Join(tmpdir, "testNetNs") - f, err := os.Create(tmpFile) - assert.NoError(err) - defer f.Close() - - err = syscall.Mount(selfNsPath, tmpFile, "bind", syscall.MS_BIND, "") - assert.Nil(err) - - isHostNs, err = hostNetworkingRequested(tmpFile) - assert.NoError(err) - assert.True(isHostNs) - - syscall.Unmount(tmpFile, 0) -} - -func TestSetupNetworkNamespace(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledNeedNonRoot) - } - - assert := assert.New(t) - - // Network namespace same as the host - config := &vc.NetworkConfig{ - NetNSPath: "/proc/self/ns/net", - } - err := setupNetworkNamespace(config) - assert.Error(err) - - // Non-existent netns path - config = &vc.NetworkConfig{ - NetNSPath: "/proc/123456789/ns/net", - } - err = setupNetworkNamespace(config) - assert.Error(err) - - // Existent netns path - n, err := ns.NewNS() - assert.NoError(err) - config = &vc.NetworkConfig{ - NetNSPath: n.Path(), - } - err = setupNetworkNamespace(config) - assert.NoError(err) - n.Close() - - // Empty netns path - config = &vc.NetworkConfig{} - err = setupNetworkNamespace(config) - assert.NoError(err) - n, err = ns.GetNS(config.NetNSPath) - assert.NoError(err) - assert.NotNil(n) - assert.True(config.NetNsCreated) - n.Close() - unix.Unmount(config.NetNSPath, unix.MNT_DETACH) - os.RemoveAll(config.NetNSPath) - - // Config with DisableNewNetNs - config = &vc.NetworkConfig{DisableNewNetNs: true} - err = setupNetworkNamespace(config) - assert.NoError(err) -} diff --git a/cli/oci.go b/cli/oci.go index 9f19a0440..2e6677cb3 100644 --- a/cli/oci.go +++ b/cli/oci.go @@ -9,15 +9,12 @@ import ( "bufio" "context" "fmt" - "io/ioutil" "net" "os" "path/filepath" - goruntime "runtime" "strings" "syscall" - "github.com/containernetworking/plugins/pkg/ns" "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/opencontainers/runc/libcontainer/utils" @@ -25,8 +22,6 @@ import ( // Contants related to cgroup memory directory const ( - ctrsMappingDirMode = os.FileMode(0750) - // Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed // here: http://man7.org/linux/man-pages/man2/statfs.2.html cgroupFsType = 0x27e0eb @@ -36,8 +31,6 @@ var cgroupsDirPath string var procMountInfo = "/proc/self/mountinfo" -var ctrsMapTreePath = "/var/run/kata-containers/containers-mapping" - // getContainerInfo returns the container status and its sandbox ID. func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStatus, string, error) { // container ID MUST be provided. @@ -45,7 +38,7 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID") } - sandboxID, err := fetchContainerIDMapping(containerID) + sandboxID, err := katautils.FetchContainerIDMapping(containerID) if err != nil { return vc.ContainerStatus{}, "", err } @@ -218,100 +211,3 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) { return cgroupRootPath, nil } - -// This function assumes it should find only one file inside the container -// ID directory. If there are several files, we could not determine which -// file name corresponds to the sandbox ID associated, and this would throw -// an error. -func fetchContainerIDMapping(containerID string) (string, error) { - if containerID == "" { - return "", fmt.Errorf("Missing container ID") - } - - dirPath := filepath.Join(ctrsMapTreePath, containerID) - - files, err := ioutil.ReadDir(dirPath) - if err != nil { - if os.IsNotExist(err) { - return "", nil - } - - return "", err - } - - if len(files) != 1 { - return "", fmt.Errorf("Too many files (%d) in %q", len(files), dirPath) - } - - return files[0].Name(), nil -} - -func addContainerIDMapping(ctx context.Context, containerID, sandboxID string) error { - span, _ := trace(ctx, "addContainerIDMapping") - defer span.Finish() - - if containerID == "" { - return fmt.Errorf("Missing container ID") - } - - if sandboxID == "" { - return fmt.Errorf("Missing sandbox ID") - } - - parentPath := filepath.Join(ctrsMapTreePath, containerID) - - if err := os.RemoveAll(parentPath); err != nil { - return err - } - - path := filepath.Join(parentPath, sandboxID) - - if err := os.MkdirAll(path, ctrsMappingDirMode); err != nil { - return err - } - - return nil -} - -func delContainerIDMapping(ctx context.Context, containerID string) error { - span, _ := trace(ctx, "delContainerIDMapping") - defer span.Finish() - - if containerID == "" { - return fmt.Errorf("Missing container ID") - } - - path := filepath.Join(ctrsMapTreePath, containerID) - - return os.RemoveAll(path) -} - -// enterNetNS is free from any call to a go routine, and it calls -// into runtime.LockOSThread(), meaning it won't be executed in a -// different thread than the one expected by the caller. -func enterNetNS(netNSPath string, cb func() error) error { - if netNSPath == "" { - return cb() - } - - goruntime.LockOSThread() - defer goruntime.UnlockOSThread() - - currentNS, err := ns.GetCurrentNS() - if err != nil { - return err - } - defer currentNS.Close() - - targetNS, err := ns.GetNS(netNSPath) - if err != nil { - return err - } - - if err := targetNS.Set(); err != nil { - return err - } - defer currentNS.Set() - - return cb() -} diff --git a/cli/oci_test.go b/cli/oci_test.go index 1fd9428d0..d2c8777c4 100644 --- a/cli/oci_test.go +++ b/cli/oci_test.go @@ -317,106 +317,3 @@ func TestGetCgroupsDirPath(t *testing.T) { assert.Equal(d.expectedResult, path) } } - -func TestFetchContainerIDMappingContainerIDEmptyFailure(t *testing.T) { - assert := assert.New(t) - - sandboxID, err := fetchContainerIDMapping("") - assert.Error(err) - assert.Empty(sandboxID) -} - -func TestFetchContainerIDMappingEmptyMappingSuccess(t *testing.T) { - assert := assert.New(t) - - path, err := ioutil.TempDir("", "containers-mapping") - assert.NoError(err) - defer os.RemoveAll(path) - ctrsMapTreePath = path - - sandboxID, err := fetchContainerIDMapping(testContainerID) - assert.NoError(err) - assert.Empty(sandboxID) -} - -func TestFetchContainerIDMappingTooManyFilesFailure(t *testing.T) { - assert := assert.New(t) - - path, err := createTempContainerIDMapping(testContainerID, testSandboxID) - assert.NoError(err) - defer os.RemoveAll(path) - err = os.MkdirAll(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID+"2"), ctrsMappingDirMode) - assert.NoError(err) - - sandboxID, err := fetchContainerIDMapping(testContainerID) - assert.Error(err) - assert.Empty(sandboxID) -} - -func TestFetchContainerIDMappingSuccess(t *testing.T) { - assert := assert.New(t) - - path, err := createTempContainerIDMapping(testContainerID, testSandboxID) - assert.NoError(err) - defer os.RemoveAll(path) - - sandboxID, err := fetchContainerIDMapping(testContainerID) - assert.NoError(err) - assert.Equal(sandboxID, testSandboxID) -} - -func TestAddContainerIDMappingContainerIDEmptyFailure(t *testing.T) { - assert := assert.New(t) - - err := addContainerIDMapping(context.Background(), "", testSandboxID) - assert.Error(err) -} - -func TestAddContainerIDMappingSandboxIDEmptyFailure(t *testing.T) { - assert := assert.New(t) - - err := addContainerIDMapping(context.Background(), testContainerID, "") - assert.Error(err) -} - -func TestAddContainerIDMappingSuccess(t *testing.T) { - assert := assert.New(t) - - path, err := ioutil.TempDir("", "containers-mapping") - assert.NoError(err) - defer os.RemoveAll(path) - ctrsMapTreePath = path - - _, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID)) - assert.True(os.IsNotExist(err)) - - err = addContainerIDMapping(context.Background(), testContainerID, testSandboxID) - assert.NoError(err) - - _, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID)) - assert.NoError(err) -} - -func TestDelContainerIDMappingContainerIDEmptyFailure(t *testing.T) { - assert := assert.New(t) - - err := delContainerIDMapping(context.Background(), "") - assert.Error(err) -} - -func TestDelContainerIDMappingSuccess(t *testing.T) { - assert := assert.New(t) - - path, err := createTempContainerIDMapping(testContainerID, testSandboxID) - assert.NoError(err) - defer os.RemoveAll(path) - - _, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID)) - assert.NoError(err) - - err = delContainerIDMapping(context.Background(), testContainerID) - assert.NoError(err) - - _, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID)) - assert.True(os.IsNotExist(err)) -} diff --git a/cli/pause.go b/cli/pause.go index 2772dd9c6..748fddfb9 100644 --- a/cli/pause.go +++ b/cli/pause.go @@ -9,6 +9,7 @@ package main import ( "context" + "github.com/kata-containers/runtime/pkg/katautils" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -57,7 +58,7 @@ func toggle(c *cli.Context, pause bool) error { } func toggleContainerPause(ctx context.Context, containerID string, pause bool) (err error) { - span, _ := trace(ctx, "pause") + span, _ := katautils.Trace(ctx, "pause") defer span.Finish() span.SetTag("pause", pause) diff --git a/cli/ps.go b/cli/ps.go index fff957ccd..26d335a24 100644 --- a/cli/ps.go +++ b/cli/ps.go @@ -10,6 +10,7 @@ import ( "context" "fmt" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -50,7 +51,7 @@ var psCLICommand = cli.Command{ } func ps(ctx context.Context, containerID, format string, args []string) error { - span, _ := trace(ctx, "ps") + span, _ := katautils.Trace(ctx, "ps") defer span.Finish() if containerID == "" { diff --git a/cli/run.go b/cli/run.go index 13a08620c..6125224e7 100644 --- a/cli/run.go +++ b/cli/run.go @@ -13,6 +13,7 @@ import ( "os" "syscall" + "github.com/kata-containers/runtime/pkg/katautils" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/urfave/cli" ) @@ -82,7 +83,7 @@ var runCLICommand = cli.Command{ func run(ctx context.Context, containerID, bundle, console, consoleSocket, pidFile string, detach, systemdCgroup bool, runtimeConfig oci.RuntimeConfig) error { - span, ctx := trace(ctx, "run") + span, ctx := katautils.Trace(ctx, "run") defer span.Finish() consolePath, err := setupConsole(console, consoleSocket) diff --git a/cli/spec.go b/cli/spec.go index a85da8e8d..3a2fc7f7e 100644 --- a/cli/spec.go +++ b/cli/spec.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "os" + "github.com/kata-containers/runtime/pkg/katautils" "github.com/opencontainers/runc/libcontainer/specconv" "github.com/urfave/cli" ) @@ -77,7 +78,7 @@ generate a proper rootless spec file.`, return err } - span, _ := trace(ctx, "spec") + span, _ := katautils.Trace(ctx, "spec") defer span.Finish() spec := specconv.Example() diff --git a/cli/start.go b/cli/start.go index d8e20664d..9ab495e32 100644 --- a/cli/start.go +++ b/cli/start.go @@ -10,6 +10,7 @@ import ( "context" "fmt" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" vcAnnot "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" @@ -48,7 +49,7 @@ var startCLICommand = cli.Command{ } func start(ctx context.Context, containerID string) (vc.VCSandbox, error) { - span, _ := trace(ctx, "start") + span, _ := katautils.Trace(ctx, "start") defer span.Finish() kataLog = kataLog.WithField("container", containerID) @@ -101,8 +102,8 @@ func start(ctx context.Context, containerID string) (vc.VCSandbox, error) { } // Run post-start OCI hooks. - err = enterNetNS(sandbox.GetNetNs(), func() error { - return postStartHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey]) + err = katautils.EnterNetNS(sandbox.GetNetNs(), func() error { + return katautils.PostStartHooks(ctx, ociSpec, sandboxID, status.Annotations[vcAnnot.BundlePathKey]) }) if err != nil { return nil, err diff --git a/cli/start_test.go b/cli/start_test.go index 5aca11362..6c4c04380 100644 --- a/cli/start_test.go +++ b/cli/start_test.go @@ -13,6 +13,7 @@ import ( "os" "testing" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" @@ -41,7 +42,7 @@ func TestStartInvalidArgs(t *testing.T) { path, err = ioutil.TempDir("", "containers-mapping") assert.NoError(err) defer os.RemoveAll(path) - ctrsMapTreePath = path + katautils.SetCtrsMapTreePath(path) // Container missing in container mapping _, err = start(context.Background(), testContainerID) diff --git a/cli/state.go b/cli/state.go index 33c7c01bf..a2fcc12e1 100644 --- a/cli/state.go +++ b/cli/state.go @@ -12,6 +12,7 @@ import ( "fmt" "os" + "github.com/kata-containers/runtime/pkg/katautils" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/urfave/cli" ) @@ -40,7 +41,7 @@ instance of a container.`, } func state(ctx context.Context, containerID string) error { - span, _ := trace(ctx, "state") + span, _ := katautils.Trace(ctx, "state") defer span.Finish() kataLog = kataLog.WithField("container", containerID) diff --git a/cli/update.go b/cli/update.go index 4b21ffbfc..91ec33ae7 100644 --- a/cli/update.go +++ b/cli/update.go @@ -13,6 +13,7 @@ import ( "strconv" "github.com/docker/go-units" + "github.com/kata-containers/runtime/pkg/katautils" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -132,7 +133,7 @@ other options are ignored. return err } - span, _ := trace(ctx, "update") + span, _ := katautils.Trace(ctx, "update") defer span.Finish() if context.Args().Present() == false { diff --git a/cli/utils.go b/cli/utils.go index 3e25c3a9f..654214791 100644 --- a/cli/utils.go +++ b/cli/utils.go @@ -8,15 +8,13 @@ package main import ( "fmt" "os" - "os/exec" "strings" "github.com/kata-containers/runtime/pkg/katautils" ) const ( - unknown = "<>" - k8sEmptyDir = "kubernetes.io~empty-dir" + unknown = "<>" ) // variables to allow tests to modify the values @@ -28,34 +26,6 @@ var ( osReleaseClr = "/usr/lib/os-release" ) -func fileExists(path string) bool { - if _, err := os.Stat(path); os.IsNotExist(err) { - return false - } - - return true -} - -// IsEphemeralStorage returns true if the given path -// to the storage belongs to kubernetes ephemeral storage -// -// This method depends on a specific path used by k8s -// to detect if it's of type ephemeral. As of now, -// this is a very k8s specific solution that works -// but in future there should be a better way for this -// method to determine if the path is for ephemeral -// volume type -func IsEphemeralStorage(path string) bool { - splitSourceSlice := strings.Split(path, "/") - if len(splitSourceSlice) > 1 { - storageType := splitSourceSlice[len(splitSourceSlice)-2] - if storageType == k8sEmptyDir { - return true - } - } - return false -} - func getKernelVersion() (string, error) { contents, err := katautils.GetFileContents(procVersion) if err != nil { @@ -151,27 +121,3 @@ func genericGetCPUDetails() (vendor, model string, err error) { return vendor, model, nil } - -// runCommandFull returns the commands space-trimmed standard output and -// error on success. Note that if the command fails, the requested output will -// still be returned, along with an error. -func runCommandFull(args []string, includeStderr bool) (string, error) { - cmd := exec.Command(args[0], args[1:]...) - var err error - var bytes []byte - - if includeStderr { - bytes, err = cmd.CombinedOutput() - } else { - bytes, err = cmd.Output() - } - - trimmed := strings.TrimSpace(string(bytes)) - - return trimmed, err -} - -// runCommand returns the commands space-trimmed standard output on success -func runCommand(args []string) (string, error) { - return runCommandFull(args, false) -} diff --git a/cli/utils_test.go b/cli/utils_test.go index 6daea066e..65c720f85 100644 --- a/cli/utils_test.go +++ b/cli/utils_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "testing" + "github.com/kata-containers/runtime/pkg/katautils" "github.com/stretchr/testify/assert" ) @@ -24,7 +25,7 @@ func TestFileExists(t *testing.T) { file := filepath.Join(dir, "foo") - assert.False(t, fileExists(file), + assert.False(t, katautils.FileExists(file), fmt.Sprintf("File %q should not exist", file)) err = createEmptyFile(file) @@ -32,24 +33,10 @@ func TestFileExists(t *testing.T) { t.Fatal(err) } - assert.True(t, fileExists(file), + assert.True(t, katautils.FileExists(file), fmt.Sprintf("File %q should exist", file)) } -func TestIsEphemeralStorage(t *testing.T) { - sampleEphePath := "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume" - isEphe := IsEphemeralStorage(sampleEphePath) - if !isEphe { - t.Fatalf("Unable to correctly determine volume type") - } - - sampleEphePath = "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/cache-volume" - isEphe = IsEphemeralStorage(sampleEphePath) - if isEphe { - t.Fatalf("Unable to correctly determine volume type") - } -} - func TestGetKernelVersion(t *testing.T) { type testData struct { contents string @@ -187,13 +174,13 @@ VERSION_ID="%s" } func TestUtilsRunCommand(t *testing.T) { - output, err := runCommand([]string{"true"}) + output, err := katautils.RunCommand([]string{"true"}) assert.NoError(t, err) assert.Equal(t, "", output) } func TestUtilsRunCommandCaptureStdout(t *testing.T) { - output, err := runCommand([]string{"echo", "hello"}) + output, err := katautils.RunCommand([]string{"echo", "hello"}) assert.NoError(t, err) assert.Equal(t, "hello", output) } @@ -201,7 +188,7 @@ func TestUtilsRunCommandCaptureStdout(t *testing.T) { func TestUtilsRunCommandIgnoreStderr(t *testing.T) { args := []string{"/bin/sh", "-c", "echo foo >&2;exit 0"} - output, err := runCommand(args) + output, err := katautils.RunCommand(args) assert.NoError(t, err) assert.Equal(t, "", output) } @@ -224,7 +211,7 @@ func TestUtilsRunCommandInvalidCmds(t *testing.T) { } for _, args := range invalidCommands { - output, err := runCommand(args) + output, err := katautils.RunCommand(args) assert.Error(t, err) assert.Equal(t, "", output) } diff --git a/cli/version.go b/cli/version.go index c4d1f695f..1862b4a1b 100644 --- a/cli/version.go +++ b/cli/version.go @@ -6,6 +6,7 @@ package main import ( + "github.com/kata-containers/runtime/pkg/katautils" "github.com/urfave/cli" ) @@ -18,7 +19,7 @@ var versionCLICommand = cli.Command{ return err } - span, _ := trace(ctx, "version") + span, _ := katautils.Trace(ctx, "version") defer span.Finish() cli.VersionPrinter(context) diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go index b05d69827..cc8116663 100644 --- a/pkg/katautils/config-settings.go +++ b/pkg/katautils/config-settings.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 @@ -15,6 +15,7 @@ var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img" var defaultFirmwarePath = "" var defaultMachineAccelerators = "" var defaultShimPath = "/usr/libexec/kata-containers/kata-shim" +var systemdUnitName = "kata-containers.target" const defaultKernelParams = "" const defaultMachineType = "pc" diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index e4bd97f99..8a12b0bc7 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 @@ -28,6 +28,9 @@ const ( var ( defaultProxy = vc.KataProxyType defaultShim = vc.KataShimType + + // if true, enable opentracing support. + tracing = false ) // The TOML configuration file contains a number of sections (or @@ -531,7 +534,7 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run return nil } -func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) { +func initConfig() (config oci.RuntimeConfig, err error) { var defaultAgentConfig interface{} defaultHypervisorConfig := vc.HypervisorConfig{ @@ -565,13 +568,6 @@ func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) { defaultAgentConfig = vc.HyperConfig{} - if builtIn { - defaultProxy = vc.KataBuiltInProxyType - defaultShim = vc.KataBuiltInShimType - - defaultAgentConfig = vc.KataAgentConfig{LongLiveConn: true} - } - config = oci.RuntimeConfig{ HypervisorType: defaultHypervisor, HypervisorConfig: defaultHypervisorConfig, @@ -592,12 +588,12 @@ func initConfig(builtIn bool) (config oci.RuntimeConfig, err error) { // // All paths are resolved fully meaning if this function does not return an // error, all paths are valid at the time of the call. -func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, tracing bool, err error) { +func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) { var resolved string - config, err = initConfig(builtIn) + config, err = initConfig() if err != nil { - return "", oci.RuntimeConfig{}, tracing, err + return "", oci.RuntimeConfig{}, err } if configPath == "" { @@ -607,18 +603,18 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved } if err != nil { - return "", config, tracing, fmt.Errorf("Cannot find usable config file (%v)", err) + return "", config, fmt.Errorf("Cannot find usable config file (%v)", err) } configData, err := ioutil.ReadFile(resolved) if err != nil { - return "", config, tracing, err + return "", config, err } var tomlConf tomlConfig _, err = toml.Decode(string(configData), &tomlConf) if err != nil { - return "", config, tracing, err + return "", config, err } config.Debug = tomlConf.Runtime.Debug @@ -633,14 +629,14 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved if tomlConf.Runtime.InterNetworkModel != "" { err = config.InterNetworkModel.SetModel(tomlConf.Runtime.InterNetworkModel) if err != nil { - return "", config, tracing, err + return "", config, err } } if !ignoreLogging { err := handleSystemLog("", "") if err != nil { - return "", config, tracing, err + return "", config, err } kataUtilsLogger.WithFields( @@ -650,13 +646,13 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved }).Info("loaded configuration") } - if err := updateRuntimeConfig(resolved, tomlConf, &config); err != nil { - return "", config, tracing, err + if err := updateConfig(resolved, tomlConf, &config, builtIn); err != nil { + return "", config, err } config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs if err := checkNetNsConfig(config); err != nil { - return "", config, tracing, err + return "", config, err } // use no proxy if HypervisorConfig.UseVSock is true @@ -667,10 +663,26 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved } if err := checkHypervisorConfig(config.HypervisorConfig); err != nil { - return "", config, tracing, err + return "", config, err } - return resolved, config, tracing, nil + return resolved, config, nil +} + +func updateConfig(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig, builtIn bool) error { + + if err := updateRuntimeConfig(configPath, tomlConf, config); err != nil { + return err + } + + if builtIn { + config.ProxyType = vc.KataBuiltInProxyType + config.ShimType = vc.KataBuiltInShimType + config.AgentType = vc.KataContainersAgent + config.AgentConfig = vc.KataAgentConfig{LongLiveConn: true} + } + + return nil } // checkNetNsConfig performs sanity checks on disable_new_netns config. diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index bd150db44..045b526e4 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 @@ -249,7 +249,7 @@ func testLoadConfiguration(t *testing.T, dir string, assert.NoError(t, err) } - resolvedConfigPath, config, _, err := LoadConfiguration(file, ignoreLogging, false) + resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false) if expectFail { assert.Error(t, err) @@ -558,7 +558,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { t.Fatal(err) } - _, config, _, err := LoadConfiguration(configPath, false, false) + _, config, err := LoadConfiguration(configPath, false, false) if err == nil { t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config) } @@ -583,7 +583,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { t.Error(err) } - _, config, _, err = LoadConfiguration(configPath, false, false) + _, config, err = LoadConfiguration(configPath, false, false) if err != nil { t.Fatal(err) } @@ -712,7 +712,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) { t.Fatal(err) } - _, config, _, err := LoadConfiguration(configPath, false, false) + _, config, err := LoadConfiguration(configPath, false, false) if err != nil { t.Fatal(err) } diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go new file mode 100644 index 000000000..5c1ab5b67 --- /dev/null +++ b/pkg/katautils/create.go @@ -0,0 +1,248 @@ +// Copyright (c) 2018 Intel Corporation +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katautils + +import ( + "context" + "fmt" + "strings" + + vc "github.com/kata-containers/runtime/virtcontainers" + vf "github.com/kata-containers/runtime/virtcontainers/factory" + "github.com/kata-containers/runtime/virtcontainers/pkg/oci" +) + +// GetKernelParamsFunc use a variable to allow tests to modify its value +var GetKernelParamsFunc = getKernelParams + +var systemdKernelParam = []vc.Param{ + { + Key: "init", + Value: "/usr/lib/systemd/systemd", + }, + { + Key: "systemd.unit", + Value: systemdUnitName, + }, + { + Key: "systemd.mask", + Value: "systemd-networkd.service", + }, + { + Key: "systemd.mask", + Value: "systemd-networkd.socket", + }, +} + +func getKernelParams(needSystemd bool) []vc.Param { + p := []vc.Param{} + + if needSystemd { + p = append(p, systemdKernelParam...) + } + + return p +} + +func needSystemd(config vc.HypervisorConfig) bool { + return config.ImagePath != "" +} + +// HandleFactory set the factory +func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeConfig) { + if !runtimeConfig.FactoryConfig.Template { + return + } + + factoryConfig := vf.Config{ + Template: true, + VMConfig: vc.VMConfig{ + HypervisorType: runtimeConfig.HypervisorType, + HypervisorConfig: runtimeConfig.HypervisorConfig, + AgentType: runtimeConfig.AgentType, + AgentConfig: runtimeConfig.AgentConfig, + }, + } + + kataUtilsLogger.WithField("factory", factoryConfig).Info("load vm factory") + + f, err := vf.NewFactory(ctx, factoryConfig, true) + if err != nil { + kataUtilsLogger.WithError(err).Warn("load vm factory failed, about to create new one") + f, err = vf.NewFactory(ctx, factoryConfig, false) + if err != nil { + kataUtilsLogger.WithError(err).Warn("create vm factory failed") + return + } + } + + vci.SetFactory(ctx, f) +} + +// SetKernelParams adds the user-specified kernel parameters (from the +// configuration file) to the defaults so that the former take priority. +func SetKernelParams(containerID string, runtimeConfig *oci.RuntimeConfig) error { + defaultKernelParams := GetKernelParamsFunc(needSystemd(runtimeConfig.HypervisorConfig)) + + if runtimeConfig.HypervisorConfig.Debug { + strParams := vc.SerializeParams(defaultKernelParams, "=") + formatted := strings.Join(strParams, " ") + + kataUtilsLogger.WithField("default-kernel-parameters", formatted).Debug() + } + + // retrieve the parameters specified in the config file + userKernelParams := runtimeConfig.HypervisorConfig.KernelParams + + // reset + runtimeConfig.HypervisorConfig.KernelParams = []vc.Param{} + + // first, add default values + for _, p := range defaultKernelParams { + if err := (runtimeConfig).AddKernelParam(p); err != nil { + return err + } + } + + // now re-add the user-specified values so that they take priority. + for _, p := range userKernelParams { + if err := (runtimeConfig).AddKernelParam(p); err != nil { + return err + } + } + + return nil +} + +// SetEphemeralStorageType sets the mount type to 'ephemeral' +// if the mount source path is provisioned by k8s for ephemeral storage. +// For the given pod ephemeral volume is created only once +// backed by tmpfs inside the VM. For successive containers +// of the same pod the already existing volume is reused. +func SetEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec { + for idx, mnt := range ociSpec.Mounts { + if IsEphemeralStorage(mnt.Source) { + ociSpec.Mounts[idx].Type = "ephemeral" + } + } + return ociSpec +} + +// CreateSandbox create a sandbox container +func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig, + containerID, bundlePath, console string, disableOutput, systemdCgroup, builtIn bool) (vc.VCSandbox, vc.Process, error) { + span, ctx := Trace(ctx, "createSandbox") + defer span.Finish() + + err := SetKernelParams(containerID, &runtimeConfig) + if err != nil { + return nil, vc.Process{}, err + } + + sandboxConfig, err := oci.SandboxConfig(ociSpec, runtimeConfig, bundlePath, containerID, console, disableOutput, systemdCgroup) + if err != nil { + return nil, vc.Process{}, err + } + + if builtIn { + sandboxConfig.Stateful = true + } + + // Important to create the network namespace before the sandbox is + // created, because it is not responsible for the creation of the + // netns if it does not exist. + if err := SetupNetworkNamespace(&sandboxConfig.NetworkConfig); err != nil { + return nil, vc.Process{}, err + } + + // Run pre-start OCI hooks. + err = EnterNetNS(sandboxConfig.NetworkConfig.NetNSPath, func() error { + return PreStartHooks(ctx, ociSpec, containerID, bundlePath) + }) + if err != nil { + return nil, vc.Process{}, err + } + + sandbox, err := vci.CreateSandbox(ctx, sandboxConfig) + if err != nil { + return nil, vc.Process{}, err + } + + sid := sandbox.ID() + kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sid) + span.SetTag("sandbox", sid) + + containers := sandbox.GetAllContainers() + if len(containers) != 1 { + return nil, vc.Process{}, fmt.Errorf("BUG: Container list from sandbox is wrong, expecting only one container, found %d containers", len(containers)) + } + + if !builtIn { + err = AddContainerIDMapping(ctx, containerID, sandbox.ID()) + if err != nil { + return nil, vc.Process{}, err + } + } + + return sandbox, containers[0].Process(), nil +} + +// CreateContainer create a container +func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) { + var c vc.VCContainer + + span, ctx := Trace(ctx, "createContainer") + defer span.Finish() + + ociSpec = SetEphemeralStorageType(ociSpec) + + contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, console, disableOutput) + if err != nil { + return vc.Process{}, err + } + + sandboxID, err := ociSpec.SandboxID() + if err != nil { + return vc.Process{}, err + } + + span.SetTag("sandbox", sandboxID) + + if builtIn { + c, err = sandbox.CreateContainer(contConfig) + if err != nil { + return vc.Process{}, err + } + } else { + kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sandboxID) + + sandbox, c, err = vci.CreateContainer(ctx, sandboxID, contConfig) + if err != nil { + return vc.Process{}, err + } + + if err := AddContainerIDMapping(ctx, containerID, sandboxID); err != nil { + return vc.Process{}, err + } + + kataUtilsLogger = kataUtilsLogger.WithField("sandbox", sandboxID) + + if err := AddContainerIDMapping(ctx, containerID, sandboxID); err != nil { + return vc.Process{}, err + } + } + + // Run pre-start OCI hooks. + err = EnterNetNS(sandbox.GetNetNs(), func() error { + return PreStartHooks(ctx, ociSpec, containerID, bundlePath) + }) + if err != nil { + return vc.Process{}, err + } + + return c.Process(), nil +} diff --git a/pkg/katautils/create_test.go b/pkg/katautils/create_test.go new file mode 100644 index 000000000..05404666d --- /dev/null +++ b/pkg/katautils/create_test.go @@ -0,0 +1,455 @@ +// Copyright (c) 2018 Intel Corporation +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katautils + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "testing" + + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" +) + +const ( + testConsole = "/dev/pts/999" + testContainerTypeAnnotation = "io.kubernetes.cri-o.ContainerType" + testSandboxIDAnnotation = "io.kubernetes.cri-o.SandboxID" + testContainerTypeContainer = "container" +) + +var ( + testBundleDir = "" + + // testingImpl is a concrete mock RVC implementation used for testing + testingImpl = &vcmock.VCMock{} +) + +// readOCIConfig returns an OCI spec. +func readOCIConfigFile(configPath string) (oci.CompatOCISpec, error) { + if configPath == "" { + return oci.CompatOCISpec{}, errors.New("BUG: need config file path") + } + + data, err := ioutil.ReadFile(configPath) + if err != nil { + return oci.CompatOCISpec{}, err + } + + var ociSpec oci.CompatOCISpec + if err := json.Unmarshal(data, &ociSpec); err != nil { + return oci.CompatOCISpec{}, err + } + caps, err := oci.ContainerCapabilities(ociSpec) + if err != nil { + return oci.CompatOCISpec{}, err + } + ociSpec.Process.Capabilities = caps + return ociSpec, nil +} + +func writeOCIConfigFile(spec oci.CompatOCISpec, configPath string) error { + if configPath == "" { + return errors.New("BUG: need config file path") + } + + bytes, err := json.MarshalIndent(spec, "", "\t") + if err != nil { + return err + } + + return ioutil.WriteFile(configPath, bytes, testFileMode) +} + +// Create an OCI bundle in the specified directory. +// +// Note that the directory will be created, but it's parent is expected to exist. +// +// This function works by copying the already-created test bundle. Ideally, +// the bundle would be recreated for each test, but createRootfs() uses +// docker which on some systems is too slow, resulting in the tests timing +// out. +func makeOCIBundle(bundleDir string) error { + from := testBundleDir + to := bundleDir + + // only the basename of bundleDir needs to exist as bundleDir + // will get created by cp(1). + base := filepath.Dir(bundleDir) + + for _, dir := range []string{from, base} { + if !FileExists(dir) { + return fmt.Errorf("BUG: directory %v should exist", dir) + } + } + + output, err := RunCommandFull([]string{"cp", "-a", from, to}, true) + if err != nil { + return fmt.Errorf("failed to copy test OCI bundle from %v to %v: %v (output: %v)", from, to, err, output) + } + + return nil +} + +// newTestRuntimeConfig creates a new RuntimeConfig +func newTestRuntimeConfig(dir, consolePath string, create bool) (oci.RuntimeConfig, error) { + if dir == "" { + return oci.RuntimeConfig{}, errors.New("BUG: need directory") + } + + hypervisorConfig, err := newTestHypervisorConfig(dir, create) + if err != nil { + return oci.RuntimeConfig{}, err + } + + return oci.RuntimeConfig{ + HypervisorType: vc.QemuHypervisor, + HypervisorConfig: hypervisorConfig, + AgentType: vc.KataContainersAgent, + ProxyType: vc.CCProxyType, + ShimType: vc.CCShimType, + Console: consolePath, + }, nil +} + +// newTestHypervisorConfig creaets a new virtcontainers +// HypervisorConfig, ensuring that the required resources are also +// created. +// +// Note: no parameter validation in case caller wishes to create an invalid +// object. +func newTestHypervisorConfig(dir string, create bool) (vc.HypervisorConfig, error) { + kernelPath := path.Join(dir, "kernel") + imagePath := path.Join(dir, "image") + hypervisorPath := path.Join(dir, "hypervisor") + + if create { + for _, file := range []string{kernelPath, imagePath, hypervisorPath} { + err := createEmptyFile(file) + if err != nil { + return vc.HypervisorConfig{}, err + } + } + } + + return vc.HypervisorConfig{ + KernelPath: kernelPath, + ImagePath: imagePath, + HypervisorPath: hypervisorPath, + HypervisorMachineType: "pc-lite", + }, nil +} + +// return the value of the *last* param with the specified key +func findLastParam(key string, params []vc.Param) (string, error) { + if key == "" { + return "", errors.New("ERROR: need non-nil key") + } + + l := len(params) + if l == 0 { + return "", errors.New("ERROR: no params") + } + + for i := l - 1; i >= 0; i-- { + p := params[i] + + if key == p.Key { + return p.Value, nil + } + } + + return "", fmt.Errorf("no param called %q found", name) +} + +func TestSetEphemeralStorageType(t *testing.T) { + assert := assert.New(t) + + ociSpec := oci.CompatOCISpec{} + var ociMounts []specs.Mount + mount := specs.Mount{ + Source: "/var/lib/kubelet/pods/366c3a77-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume", + } + + ociMounts = append(ociMounts, mount) + ociSpec.Mounts = ociMounts + ociSpec = SetEphemeralStorageType(ociSpec) + + mountType := ociSpec.Mounts[0].Type + assert.Equal(mountType, "ephemeral", + "Unexpected mount type, got %s expected ephemeral", mountType) +} + +func TestSetKernelParams(t *testing.T) { + assert := assert.New(t) + + config := oci.RuntimeConfig{} + + assert.Empty(config.HypervisorConfig.KernelParams) + + err := SetKernelParams(testContainerID, &config) + assert.NoError(err) + + if needSystemd(config.HypervisorConfig) { + assert.NotEmpty(config.HypervisorConfig.KernelParams) + } +} + +func TestSetKernelParamsUserOptionTakesPriority(t *testing.T) { + assert := assert.New(t) + + initName := "init" + initValue := "/sbin/myinit" + + ipName := "ip" + ipValue := "127.0.0.1" + + params := []vc.Param{ + {Key: initName, Value: initValue}, + {Key: ipName, Value: ipValue}, + } + + hypervisorConfig := vc.HypervisorConfig{ + KernelParams: params, + } + + // Config containing user-specified kernel parameters + config := oci.RuntimeConfig{ + HypervisorConfig: hypervisorConfig, + } + + assert.NotEmpty(config.HypervisorConfig.KernelParams) + + err := SetKernelParams(testContainerID, &config) + assert.NoError(err) + + kernelParams := config.HypervisorConfig.KernelParams + + init, err := findLastParam(initName, kernelParams) + assert.NoError(err) + assert.Equal(initValue, init) + + ip, err := findLastParam(ipName, kernelParams) + assert.NoError(err) + assert.Equal(ipValue, ip) + +} + +func TestCreateSandboxConfigFail(t *testing.T) { + assert := assert.New(t) + + path, err := ioutil.TempDir("", "containers-mapping") + assert.NoError(err) + defer os.RemoveAll(path) + ctrsMapTreePath = path + + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) + assert.NoError(err) + + bundlePath := filepath.Join(tmpdir, "bundle") + + err = makeOCIBundle(bundlePath) + assert.NoError(err) + + ociConfigFile := filepath.Join(bundlePath, "config.json") + assert.True(FileExists(ociConfigFile)) + + spec, err := readOCIConfigFile(ociConfigFile) + assert.NoError(err) + + quota := int64(0) + limit := int64(0) + + spec.Linux.Resources.Memory = &specs.LinuxMemory{ + Limit: &limit, + } + + spec.Linux.Resources.CPU = &specs.LinuxCPU{ + // specify an invalid value + Quota: "a, + } + + _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false) + assert.Error(err) +} + +func TestCreateSandboxFail(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledNeedNonRoot) + } + + assert := assert.New(t) + + path, err := ioutil.TempDir("", "containers-mapping") + assert.NoError(err) + defer os.RemoveAll(path) + ctrsMapTreePath = path + + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + runtimeConfig, err := newTestRuntimeConfig(tmpdir, testConsole, true) + assert.NoError(err) + + bundlePath := filepath.Join(tmpdir, "bundle") + + err = makeOCIBundle(bundlePath) + assert.NoError(err) + + ociConfigFile := filepath.Join(bundlePath, "config.json") + assert.True(FileExists(ociConfigFile)) + + spec, err := readOCIConfigFile(ociConfigFile) + assert.NoError(err) + + _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false) + assert.Error(err) + assert.True(vcmock.IsMockError(err)) +} + +func TestCreateContainerContainerConfigFail(t *testing.T) { + assert := assert.New(t) + + path, err := ioutil.TempDir("", "containers-mapping") + assert.NoError(err) + defer os.RemoveAll(path) + ctrsMapTreePath = path + + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + bundlePath := filepath.Join(tmpdir, "bundle") + + err = makeOCIBundle(bundlePath) + assert.NoError(err) + + ociConfigFile := filepath.Join(bundlePath, "config.json") + assert.True(FileExists(ociConfigFile)) + + spec, err := readOCIConfigFile(ociConfigFile) + assert.NoError(err) + + // Set invalid container type + containerType := "你好，世界" + spec.Annotations = make(map[string]string) + spec.Annotations[testContainerTypeAnnotation] = containerType + + // rewrite file + err = writeOCIConfigFile(spec, ociConfigFile) + assert.NoError(err) + + for _, disableOutput := range []bool{true, false} { + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + assert.Error(err) + assert.False(vcmock.IsMockError(err)) + assert.True(strings.Contains(err.Error(), containerType)) + os.RemoveAll(path) + } +} + +func TestCreateContainerFail(t *testing.T) { + assert := assert.New(t) + + path, err := ioutil.TempDir("", "containers-mapping") + assert.NoError(err) + defer os.RemoveAll(path) + ctrsMapTreePath = path + + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + bundlePath := filepath.Join(tmpdir, "bundle") + + err = makeOCIBundle(bundlePath) + assert.NoError(err) + + ociConfigFile := filepath.Join(bundlePath, "config.json") + assert.True(FileExists(ociConfigFile)) + + spec, err := readOCIConfigFile(ociConfigFile) + assert.NoError(err) + + // set expected container type and sandboxID + spec.Annotations = make(map[string]string) + spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer + spec.Annotations[testSandboxIDAnnotation] = testSandboxID + + // rewrite file + err = writeOCIConfigFile(spec, ociConfigFile) + assert.NoError(err) + + for _, disableOutput := range []bool{true, false} { + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + assert.Error(err) + assert.True(vcmock.IsMockError(err)) + os.RemoveAll(path) + } +} + +func TestCreateContainer(t *testing.T) { + assert := assert.New(t) + + path, err := ioutil.TempDir("", "containers-mapping") + assert.NoError(err) + defer os.RemoveAll(path) + ctrsMapTreePath = path + + testingImpl.CreateContainerFunc = func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) { + return &vcmock.Sandbox{}, &vcmock.Container{}, nil + } + + defer func() { + testingImpl.CreateContainerFunc = nil + }() + + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + bundlePath := filepath.Join(tmpdir, "bundle") + + err = makeOCIBundle(bundlePath) + assert.NoError(err) + + ociConfigFile := filepath.Join(bundlePath, "config.json") + assert.True(FileExists(ociConfigFile)) + + spec, err := readOCIConfigFile(ociConfigFile) + assert.NoError(err) + + // set expected container type and sandboxID + spec.Annotations = make(map[string]string) + spec.Annotations[testContainerTypeAnnotation] = testContainerTypeContainer + spec.Annotations[testSandboxIDAnnotation] = testSandboxID + + // rewrite file + err = writeOCIConfigFile(spec, ociConfigFile) + assert.NoError(err) + + for _, disableOutput := range []bool{true, false} { + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + assert.NoError(err) + os.RemoveAll(path) + } +} diff --git a/cli/hook.go b/pkg/katautils/hook.go similarity index 84% rename from cli/hook.go rename to pkg/katautils/hook.go index b5f96fe74..cfb15522e 100644 --- a/cli/hook.go +++ b/pkg/katautils/hook.go @@ -1,9 +1,10 @@ // Copyright (c) 2018 Intel Corporation +// Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 // -package main +package katautils import ( "bytes" @@ -24,11 +25,11 @@ import ( // Logger returns a logrus logger appropriate for logging hook messages func hookLogger() *logrus.Entry { - return kataLog.WithField("subsystem", "hook") + return kataUtilsLogger.WithField("subsystem", "hook") } func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error { - span, _ := trace(ctx, "hook") + span, _ := Trace(ctx, "hook") defer span.Finish() span.SetTag("subsystem", "runHook") @@ -91,7 +92,7 @@ func runHook(ctx context.Context, hook specs.Hook, cid, bundlePath string) error } func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType string) error { - span, _ := trace(ctx, "hooks") + span, _ := Trace(ctx, "hooks") defer span.Finish() span.SetTag("subsystem", hookType) @@ -110,7 +111,8 @@ func runHooks(ctx context.Context, hooks []specs.Hook, cid, bundlePath, hookType return nil } -func preStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error { +// PreStartHooks run the hooks before start container +func PreStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error { // If no hook available, nothing needs to be done. if spec.Hooks == nil { return nil @@ -119,7 +121,8 @@ func preStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath return runHooks(ctx, spec.Hooks.Prestart, cid, bundlePath, "pre-start") } -func postStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error { +// PostStartHooks run the hooks just after start container +func PostStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error { // If no hook available, nothing needs to be done. if spec.Hooks == nil { return nil @@ -128,7 +131,8 @@ func postStartHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath return runHooks(ctx, spec.Hooks.Poststart, cid, bundlePath, "post-start") } -func postStopHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error { +// PostStopHooks run the hooks after stop container +func PostStopHooks(ctx context.Context, spec oci.CompatOCISpec, cid, bundlePath string) error { // If no hook available, nothing needs to be done. if spec.Hooks == nil { return nil diff --git a/cli/hook_test.go b/pkg/katautils/hook_test.go similarity index 85% rename from cli/hook_test.go rename to pkg/katautils/hook_test.go index 0f3fed8c5..8bc09142e 100644 --- a/cli/hook_test.go +++ b/pkg/katautils/hook_test.go @@ -1,9 +1,10 @@ // Copyright (c) 2018 Intel Corporation +// Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 // -package main +package katautils import ( "context" @@ -95,7 +96,7 @@ func TestPreStartHooks(t *testing.T) { // Hooks field is nil spec := oci.CompatOCISpec{} - err := preStartHooks(ctx, spec, "", "") + err := PreStartHooks(ctx, spec, "", "") assert.NoError(err) // Hooks list is empty @@ -104,7 +105,7 @@ func TestPreStartHooks(t *testing.T) { Hooks: &specs.Hooks{}, }, } - err = preStartHooks(ctx, spec, "", "") + err = PreStartHooks(ctx, spec, "", "") assert.NoError(err) // Run with timeout 0 @@ -116,7 +117,7 @@ func TestPreStartHooks(t *testing.T) { }, }, } - err = preStartHooks(ctx, spec, testSandboxID, testBundlePath) + err = PreStartHooks(ctx, spec, testSandboxID, testBundlePath) assert.NoError(err) // Failure due to wrong hook @@ -128,7 +129,7 @@ func TestPreStartHooks(t *testing.T) { }, }, } - err = preStartHooks(ctx, spec, testSandboxID, testBundlePath) + err = PreStartHooks(ctx, spec, testSandboxID, testBundlePath) assert.Error(err) } @@ -143,7 +144,7 @@ func TestPostStartHooks(t *testing.T) { // Hooks field is nil spec := oci.CompatOCISpec{} - err := postStartHooks(ctx, spec, "", "") + err := PostStartHooks(ctx, spec, "", "") assert.NoError(err) // Hooks list is empty @@ -152,7 +153,7 @@ func TestPostStartHooks(t *testing.T) { Hooks: &specs.Hooks{}, }, } - err = postStartHooks(ctx, spec, "", "") + err = PostStartHooks(ctx, spec, "", "") assert.NoError(err) // Run with timeout 0 @@ -164,7 +165,7 @@ func TestPostStartHooks(t *testing.T) { }, }, } - err = postStartHooks(ctx, spec, testSandboxID, testBundlePath) + err = PostStartHooks(ctx, spec, testSandboxID, testBundlePath) assert.NoError(err) // Failure due to wrong hook @@ -176,7 +177,7 @@ func TestPostStartHooks(t *testing.T) { }, }, } - err = postStartHooks(ctx, spec, testSandboxID, testBundlePath) + err = PostStartHooks(ctx, spec, testSandboxID, testBundlePath) assert.Error(err) } @@ -191,7 +192,7 @@ func TestPostStopHooks(t *testing.T) { // Hooks field is nil spec := oci.CompatOCISpec{} - err := postStopHooks(ctx, spec, "", "") + err := PostStopHooks(ctx, spec, "", "") assert.NoError(err) // Hooks list is empty @@ -200,7 +201,7 @@ func TestPostStopHooks(t *testing.T) { Hooks: &specs.Hooks{}, }, } - err = postStopHooks(ctx, spec, "", "") + err = PostStopHooks(ctx, spec, "", "") assert.NoError(err) // Run with timeout 0 @@ -212,7 +213,7 @@ func TestPostStopHooks(t *testing.T) { }, }, } - err = postStopHooks(ctx, spec, testSandboxID, testBundlePath) + err = PostStopHooks(ctx, spec, testSandboxID, testBundlePath) assert.NoError(err) // Failure due to wrong hook @@ -224,6 +225,6 @@ func TestPostStopHooks(t *testing.T) { }, }, } - err = postStopHooks(ctx, spec, testSandboxID, testBundlePath) + err = PostStopHooks(ctx, spec, testSandboxID, testBundlePath) assert.Error(err) } diff --git a/pkg/katautils/logger.go b/pkg/katautils/logger.go index 544be045f..13d9ab5bd 100644 --- a/pkg/katautils/logger.go +++ b/pkg/katautils/logger.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 diff --git a/pkg/katautils/logger_test.go b/pkg/katautils/logger_test.go index 9367eb680..ff8720fc1 100644 --- a/pkg/katautils/logger_test.go +++ b/pkg/katautils/logger_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 diff --git a/pkg/katautils/network.go b/pkg/katautils/network.go new file mode 100644 index 000000000..036b611db --- /dev/null +++ b/pkg/katautils/network.go @@ -0,0 +1,181 @@ +// Copyright (c) 2018 Intel Corporation +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katautils + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + goruntime "runtime" + "strings" + + "github.com/containernetworking/plugins/pkg/ns" + vc "github.com/kata-containers/runtime/virtcontainers" + "golang.org/x/sys/unix" +) + +const procMountInfoFile = "/proc/self/mountinfo" + +// EnterNetNS is free from any call to a go routine, and it calls +// into runtime.LockOSThread(), meaning it won't be executed in a +// different thread than the one expected by the caller. +func EnterNetNS(netNSPath string, cb func() error) error { + if netNSPath == "" { + return cb() + } + + goruntime.LockOSThread() + defer goruntime.UnlockOSThread() + + currentNS, err := ns.GetCurrentNS() + if err != nil { + return err + } + defer currentNS.Close() + + targetNS, err := ns.GetNS(netNSPath) + if err != nil { + return err + } + + if err := targetNS.Set(); err != nil { + return err + } + defer currentNS.Set() + + return cb() +} + +// SetupNetworkNamespace create a network namespace +func SetupNetworkNamespace(config *vc.NetworkConfig) error { + if config.DisableNewNetNs { + kataUtilsLogger.Info("DisableNewNetNs is on, shim and hypervisor are running in the host netns") + return nil + } + + if config.NetNSPath == "" { + n, err := ns.NewNS() + if err != nil { + return err + } + + config.NetNSPath = n.Path() + config.NetNsCreated = true + + return nil + } + + isHostNs, err := hostNetworkingRequested(config.NetNSPath) + if err != nil { + return err + } + if isHostNs { + return fmt.Errorf("Host networking requested, not supported by runtime") + } + + return nil +} + +// getNetNsFromBindMount returns the network namespace for the bind-mounted path +func getNetNsFromBindMount(nsPath string, procMountFile string) (string, error) { + netNsMountType := "nsfs" + + // Resolve all symlinks in the path as the mountinfo file contains + // resolved paths. + nsPath, err := filepath.EvalSymlinks(nsPath) + if err != nil { + return "", err + } + + f, err := os.Open(procMountFile) + if err != nil { + return "", err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + text := scanner.Text() + + // Scan the mountinfo file to search for the network namespace path + // This file contains mounts in the eg format: + // "711 26 0:3 net:[4026532009] /run/docker/netns/default rw shared:535 - nsfs nsfs rw" + // + // Reference: https://www.kernel.org/doc/Documentation/filesystems/proc.txt + + // We are interested in the first 9 fields of this file, + // to check for the correct mount type. + fields := strings.Split(text, " ") + if len(fields) < 9 { + continue + } + + // We check here if the mount type is a network namespace mount type, namely "nsfs" + mountTypeFieldIdx := 8 + if fields[mountTypeFieldIdx] != netNsMountType { + continue + } + + // This is the mount point/destination for the mount + mntDestIdx := 4 + if fields[mntDestIdx] != nsPath { + continue + } + + // This is the root/source of the mount + return fields[3], nil + } + + return "", nil +} + +// hostNetworkingRequested checks if the network namespace requested is the +// same as the current process. +func hostNetworkingRequested(configNetNs string) (bool, error) { + var evalNS, nsPath, currentNsPath string + var err error + + // Net namespace provided as "/proc/pid/ns/net" or "/proc//task//ns/net" + if strings.HasPrefix(configNetNs, "/proc") && strings.HasSuffix(configNetNs, "/ns/net") { + if _, err := os.Stat(configNetNs); err != nil { + return false, err + } + + // Here we are trying to resolve the path but it fails because + // namespaces links don't really exist. For this reason, the + // call to EvalSymlinks will fail when it will try to stat the + // resolved path found. As we only care about the path, we can + // retrieve it from the PathError structure. + if _, err = filepath.EvalSymlinks(configNetNs); err != nil { + nsPath = err.(*os.PathError).Path + } else { + return false, fmt.Errorf("Net namespace path %s is not a symlink", configNetNs) + } + + _, evalNS = filepath.Split(nsPath) + + } else { + // Bind-mounted path provided + evalNS, _ = getNetNsFromBindMount(configNetNs, procMountInfoFile) + } + + currentNS := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid()) + if _, err = filepath.EvalSymlinks(currentNS); err != nil { + currentNsPath = err.(*os.PathError).Path + } else { + return false, fmt.Errorf("Unexpected: Current network namespace path is not a symlink") + } + + _, evalCurrentNS := filepath.Split(currentNsPath) + + if evalNS == evalCurrentNS { + return true, nil + } + + return false, nil +} diff --git a/pkg/katautils/network_test.go b/pkg/katautils/network_test.go new file mode 100644 index 000000000..857568ce5 --- /dev/null +++ b/pkg/katautils/network_test.go @@ -0,0 +1,150 @@ +// Copyright (c) 2018 Huawei Corporation. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katautils + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/containernetworking/plugins/pkg/ns" + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/unix" +) + +func TestGetNetNsFromBindMount(t *testing.T) { + assert := assert.New(t) + + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + mountFile := filepath.Join(tmpdir, "mountInfo") + nsPath := filepath.Join(tmpdir, "ns123") + + // Non-existent namespace path + _, err = getNetNsFromBindMount(nsPath, mountFile) + assert.NotNil(err) + + tmpNSPath := filepath.Join(tmpdir, "testNetNs") + f, err := os.Create(tmpNSPath) + assert.NoError(err) + defer f.Close() + + type testData struct { + contents string + expectedResult string + } + + data := []testData{ + {fmt.Sprintf("711 26 0:3 net:[4026532008] %s rw shared:535 - nsfs nsfs rw", tmpNSPath), "net:[4026532008]"}, + {"711 26 0:3 net:[4026532008] /run/netns/ns123 rw shared:535 - tmpfs tmpfs rw", ""}, + {"a a a a a a a - b c d", ""}, + {"", ""}, + } + + for i, d := range data { + err := ioutil.WriteFile(mountFile, []byte(d.contents), 0640) + assert.NoError(err) + + path, err := getNetNsFromBindMount(tmpNSPath, mountFile) + assert.NoError(err, fmt.Sprintf("got %q, test data: %+v", path, d)) + + assert.Equal(d.expectedResult, path, "Test %d, expected %s, got %s", i, d.expectedResult, path) + } +} + +func TestHostNetworkingRequested(t *testing.T) { + assert := assert.New(t) + + if os.Geteuid() != 0 { + t.Skip(testDisabledNeedRoot) + } + + // Network namespace same as the host + selfNsPath := "/proc/self/ns/net" + isHostNs, err := hostNetworkingRequested(selfNsPath) + assert.NoError(err) + assert.True(isHostNs) + + // Non-existent netns path + nsPath := "/proc/123456789/ns/net" + _, err = hostNetworkingRequested(nsPath) + assert.Error(err) + + // Bind-mounted Netns + tmpdir, err := ioutil.TempDir("", "") + assert.NoError(err) + defer os.RemoveAll(tmpdir) + + // Create a bind mount to the current network namespace. + tmpFile := filepath.Join(tmpdir, "testNetNs") + f, err := os.Create(tmpFile) + assert.NoError(err) + defer f.Close() + + err = syscall.Mount(selfNsPath, tmpFile, "bind", syscall.MS_BIND, "") + assert.Nil(err) + + isHostNs, err = hostNetworkingRequested(tmpFile) + assert.NoError(err) + assert.True(isHostNs) + + syscall.Unmount(tmpFile, 0) +} + +func TestSetupNetworkNamespace(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledNeedNonRoot) + } + + assert := assert.New(t) + + // Network namespace same as the host + config := &vc.NetworkConfig{ + NetNSPath: "/proc/self/ns/net", + } + err := SetupNetworkNamespace(config) + assert.Error(err) + + // Non-existent netns path + config = &vc.NetworkConfig{ + NetNSPath: "/proc/123456789/ns/net", + } + err = SetupNetworkNamespace(config) + assert.Error(err) + + // Existent netns path + n, err := ns.NewNS() + assert.NoError(err) + config = &vc.NetworkConfig{ + NetNSPath: n.Path(), + } + err = SetupNetworkNamespace(config) + assert.NoError(err) + n.Close() + + // Empty netns path + config = &vc.NetworkConfig{} + err = SetupNetworkNamespace(config) + assert.NoError(err) + n, err = ns.GetNS(config.NetNSPath) + assert.NoError(err) + assert.NotNil(n) + assert.True(config.NetNsCreated) + n.Close() + unix.Unmount(config.NetNSPath, unix.MNT_DETACH) + os.RemoveAll(config.NetNSPath) + + // Config with DisableNewNetNs + config = &vc.NetworkConfig{DisableNewNetNs: true} + err = SetupNetworkNamespace(config) + assert.NoError(err) +} diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go new file mode 100644 index 000000000..0eb80d584 --- /dev/null +++ b/pkg/katautils/oci.go @@ -0,0 +1,92 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katautils + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +const ctrsMappingDirMode = os.FileMode(0750) + +var ctrsMapTreePath = "/var/run/kata-containers/containers-mapping" + +// SetCtrsMapTreePath let the testcases change the ctrsMapTreePath to a test dir +func SetCtrsMapTreePath(path string) { + ctrsMapTreePath = path +} + +// FetchContainerIDMapping This function assumes it should find only one file inside the container +// ID directory. If there are several files, we could not determine which +// file name corresponds to the sandbox ID associated, and this would throw +// an error. +func FetchContainerIDMapping(containerID string) (string, error) { + if containerID == "" { + return "", fmt.Errorf("Missing container ID") + } + + dirPath := filepath.Join(ctrsMapTreePath, containerID) + + files, err := ioutil.ReadDir(dirPath) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + + return "", err + } + + if len(files) != 1 { + return "", fmt.Errorf("Too many files (%d) in %q", len(files), dirPath) + } + + return files[0].Name(), nil +} + +// AddContainerIDMapping add a container id mapping to sandbox id +func AddContainerIDMapping(ctx context.Context, containerID, sandboxID string) error { + span, _ := Trace(ctx, "addContainerIDMapping") + defer span.Finish() + + if containerID == "" { + return fmt.Errorf("Missing container ID") + } + + if sandboxID == "" { + return fmt.Errorf("Missing sandbox ID") + } + + parentPath := filepath.Join(ctrsMapTreePath, containerID) + + if err := os.RemoveAll(parentPath); err != nil { + return err + } + + path := filepath.Join(parentPath, sandboxID) + + if err := os.MkdirAll(path, ctrsMappingDirMode); err != nil { + return err + } + + return nil +} + +// DelContainerIDMapping delete container id mapping from a sandbox +func DelContainerIDMapping(ctx context.Context, containerID string) error { + span, _ := Trace(ctx, "delContainerIDMapping") + defer span.Finish() + + if containerID == "" { + return fmt.Errorf("Missing container ID") + } + + path := filepath.Join(ctrsMapTreePath, containerID) + + return os.RemoveAll(path) +} diff --git a/pkg/katautils/oci_test.go b/pkg/katautils/oci_test.go new file mode 100644 index 000000000..1d19485d2 --- /dev/null +++ b/pkg/katautils/oci_test.go @@ -0,0 +1,135 @@ +// Copyright (c) 2018 Intel Corporation +// Copyright (c) 2018 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package katautils + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func createTempContainerIDMapping(containerID, sandboxID string) (string, error) { + tmpDir, err := ioutil.TempDir("", "containers-mapping") + if err != nil { + return "", err + } + ctrsMapTreePath = tmpDir + + path := filepath.Join(ctrsMapTreePath, containerID, sandboxID) + if err := os.MkdirAll(path, 0750); err != nil { + return "", err + } + + return tmpDir, nil +} + +func TestFetchContainerIDMappingContainerIDEmptyFailure(t *testing.T) { + assert := assert.New(t) + + sandboxID, err := FetchContainerIDMapping("") + assert.Error(err) + assert.Empty(sandboxID) +} + +func TestFetchContainerIDMappingEmptyMappingSuccess(t *testing.T) { + assert := assert.New(t) + + path, err := ioutil.TempDir("", "containers-mapping") + assert.NoError(err) + defer os.RemoveAll(path) + ctrsMapTreePath = path + + sandboxID, err := FetchContainerIDMapping(testContainerID) + assert.NoError(err) + assert.Empty(sandboxID) +} + +func TestFetchContainerIDMappingTooManyFilesFailure(t *testing.T) { + assert := assert.New(t) + + path, err := createTempContainerIDMapping(testContainerID, testSandboxID) + assert.NoError(err) + defer os.RemoveAll(path) + err = os.MkdirAll(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID+"2"), ctrsMappingDirMode) + assert.NoError(err) + + sandboxID, err := FetchContainerIDMapping(testContainerID) + assert.Error(err) + assert.Empty(sandboxID) +} + +func TestFetchContainerIDMappingSuccess(t *testing.T) { + assert := assert.New(t) + + path, err := createTempContainerIDMapping(testContainerID, testSandboxID) + assert.NoError(err) + defer os.RemoveAll(path) + + sandboxID, err := FetchContainerIDMapping(testContainerID) + assert.NoError(err) + assert.Equal(sandboxID, testSandboxID) +} + +func TestAddContainerIDMappingContainerIDEmptyFailure(t *testing.T) { + assert := assert.New(t) + + err := AddContainerIDMapping(context.Background(), "", testSandboxID) + assert.Error(err) +} + +func TestAddContainerIDMappingSandboxIDEmptyFailure(t *testing.T) { + assert := assert.New(t) + + err := AddContainerIDMapping(context.Background(), testContainerID, "") + assert.Error(err) +} + +func TestAddContainerIDMappingSuccess(t *testing.T) { + assert := assert.New(t) + + path, err := ioutil.TempDir("", "containers-mapping") + assert.NoError(err) + defer os.RemoveAll(path) + ctrsMapTreePath = path + + _, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID)) + assert.True(os.IsNotExist(err)) + + err = AddContainerIDMapping(context.Background(), testContainerID, testSandboxID) + assert.NoError(err) + + _, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID)) + assert.NoError(err) +} + +func TestDelContainerIDMappingContainerIDEmptyFailure(t *testing.T) { + assert := assert.New(t) + + err := DelContainerIDMapping(context.Background(), "") + assert.Error(err) +} + +func TestDelContainerIDMappingSuccess(t *testing.T) { + assert := assert.New(t) + + path, err := createTempContainerIDMapping(testContainerID, testSandboxID) + assert.NoError(err) + defer os.RemoveAll(path) + + _, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID)) + assert.NoError(err) + + err = DelContainerIDMapping(context.Background(), testContainerID) + assert.NoError(err) + + _, err = os.Stat(filepath.Join(ctrsMapTreePath, testContainerID, testSandboxID)) + assert.True(os.IsNotExist(err)) +} diff --git a/cli/tracing.go b/pkg/katautils/tracing.go similarity index 77% rename from cli/tracing.go rename to pkg/katautils/tracing.go index 92cbbc605..10d95ce3f 100644 --- a/cli/tracing.go +++ b/pkg/katautils/tracing.go @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 // -package main +package katautils import ( "context" @@ -22,14 +22,15 @@ type traceLogger struct { var tracerCloser io.Closer func (t traceLogger) Error(msg string) { - kataLog.Error(msg) + kataUtilsLogger.Error(msg) } func (t traceLogger) Infof(msg string, args ...interface{}) { - kataLog.Infof(msg, args...) + kataUtilsLogger.Infof(msg, args...) } -func createTracer(name string) (opentracing.Tracer, error) { +// CreateTracer create a tracer +func CreateTracer(name string) (opentracing.Tracer, error) { cfg := &config.Configuration{ ServiceName: name, @@ -61,8 +62,8 @@ func createTracer(name string) (opentracing.Tracer, error) { return tracer, nil } -// stopTracing() ends all tracing, reporting the spans to the collector. -func stopTracing(ctx context.Context) { +// StopTracing ends all tracing, reporting the spans to the collector. +func StopTracing(ctx context.Context) { if !tracing { return } @@ -74,12 +75,14 @@ func stopTracing(ctx context.Context) { } // report all possible spans to the collector - tracerCloser.Close() + if tracerCloser != nil { + tracerCloser.Close() + } } -// trace creates a new tracing span based on the specified name and parent +// Trace creates a new tracing span based on the specified name and parent // context. -func trace(parent context.Context, name string) (opentracing.Span, context.Context) { +func Trace(parent context.Context, name string) (opentracing.Span, context.Context) { span, ctx := opentracing.StartSpanFromContext(parent, name) span.SetTag("source", "runtime") diff --git a/pkg/katautils/utils.go b/pkg/katautils/utils.go index 699ead9e8..a8c1ca96f 100644 --- a/pkg/katautils/utils.go +++ b/pkg/katautils/utils.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 @@ -10,10 +10,45 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "path/filepath" + "strings" "syscall" ) +const ( + k8sEmptyDir = "kubernetes.io~empty-dir" +) + +// FileExists test is a file exiting or not +func FileExists(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + + return true +} + +// IsEphemeralStorage returns true if the given path +// to the storage belongs to kubernetes ephemeral storage +// +// This method depends on a specific path used by k8s +// to detect if it's of type ephemeral. As of now, +// this is a very k8s specific solution that works +// but in future there should be a better way for this +// method to determine if the path is for ephemeral +// volume type +func IsEphemeralStorage(path string) bool { + splitSourceSlice := strings.Split(path, "/") + if len(splitSourceSlice) > 1 { + storageType := splitSourceSlice[len(splitSourceSlice)-2] + if storageType == k8sEmptyDir { + return true + } + } + return false +} + // ResolvePath returns the fully resolved and expanded value of the // specified path. func ResolvePath(path string) (string, error) { @@ -75,3 +110,27 @@ func GetFileContents(file string) (string, error) { return string(bytes), nil } + +// RunCommandFull returns the commands space-trimmed standard output and +// error on success. Note that if the command fails, the requested output will +// still be returned, along with an error. +func RunCommandFull(args []string, includeStderr bool) (string, error) { + cmd := exec.Command(args[0], args[1:]...) + var err error + var bytes []byte + + if includeStderr { + bytes, err = cmd.CombinedOutput() + } else { + bytes, err = cmd.Output() + } + + trimmed := strings.TrimSpace(string(bytes)) + + return trimmed, err +} + +// RunCommand returns the commands space-trimmed standard output on success +func RunCommand(args []string) (string, error) { + return RunCommandFull(args, false) +} diff --git a/pkg/katautils/utils_test.go b/pkg/katautils/utils_test.go index a10f2c867..d061e6dc0 100644 --- a/pkg/katautils/utils_test.go +++ b/pkg/katautils/utils_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Intel Corporation +// Copyright (c) 2018 Intel Corporation // Copyright (c) 2018 HyperHQ Inc. // // SPDX-License-Identifier: Apache-2.0 @@ -7,14 +7,18 @@ package katautils import ( + "errors" "fmt" "io/ioutil" "os" + "os/exec" "path" "path/filepath" + "strings" "syscall" "testing" + "github.com/kata-containers/runtime/virtcontainers/pkg/oci" "github.com/stretchr/testify/assert" ) @@ -22,7 +26,16 @@ const ( testDirMode = os.FileMode(0750) testFileMode = os.FileMode(0640) + testDisabledNeedRoot = "Test disabled as requires root user" testDisabledNeedNonRoot = "Test disabled as requires non-root user" + + // small docker image used to create root filesystems from + testDockerImage = "busybox" + + testSandboxID = "99999999-9999-9999-99999999999999999" + testContainerID = "1" + testBundle = "bundle" + specConfig = "config.json" ) var testDir = "" @@ -37,6 +50,148 @@ func init() { } fmt.Printf("INFO: test directory is %v\n", testDir) + + testBundleDir = filepath.Join(testDir, testBundle) + err = os.MkdirAll(testBundleDir, testDirMode) + if err != nil { + panic(fmt.Sprintf("ERROR: failed to create bundle directory %v: %v", testBundleDir, err)) + } + + fmt.Printf("INFO: creating OCI bundle in %v for tests to use\n", testBundleDir) + err = realMakeOCIBundle(testBundleDir) + if err != nil { + panic(fmt.Sprintf("ERROR: failed to create OCI bundle: %v", err)) + } +} + +// createOCIConfig creates an OCI configuration (spec) file in +// the bundle directory specified (which must exist). +func createOCIConfig(bundleDir string) error { + if bundleDir == "" { + return errors.New("BUG: Need bundle directory") + } + + if !FileExists(bundleDir) { + return fmt.Errorf("BUG: Bundle directory %s does not exist", bundleDir) + } + + var configCmd string + + // Search for a suitable version of runc to use to generate + // the OCI config file. + for _, cmd := range []string{"docker-runc", "runc"} { + fullPath, err := exec.LookPath(cmd) + if err == nil { + configCmd = fullPath + break + } + } + + if configCmd == "" { + return errors.New("Cannot find command to generate OCI config file") + } + + _, err := RunCommand([]string{configCmd, "spec", "--bundle", bundleDir}) + if err != nil { + return err + } + + specFile := filepath.Join(bundleDir, specConfig) + if !FileExists(specFile) { + return fmt.Errorf("generated OCI config file does not exist: %v", specFile) + } + + return nil +} + +// realMakeOCIBundle will create an OCI bundle (including the "config.json" +// config file) in the directory specified (which must already exist). +// +// XXX: Note that tests should *NOT* call this function - they should +// XXX: instead call makeOCIBundle(). +func realMakeOCIBundle(bundleDir string) error { + if bundleDir == "" { + return errors.New("BUG: Need bundle directory") + } + + if !FileExists(bundleDir) { + return fmt.Errorf("BUG: Bundle directory %v does not exist", bundleDir) + } + + err := createOCIConfig(bundleDir) + if err != nil { + return err + } + + // Note the unusual parameter (a directory, not the config + // file to parse!) + spec, err := oci.ParseConfigJSON(bundleDir) + if err != nil { + return err + } + + // Determine the rootfs directory name the OCI config refers to + ociRootPath := spec.Root.Path + + rootfsDir := filepath.Join(bundleDir, ociRootPath) + + if strings.HasPrefix(ociRootPath, "/") { + return fmt.Errorf("Cannot handle absolute rootfs as bundle must be unique to each test") + } + + err = createRootfs(rootfsDir) + if err != nil { + return err + } + + return nil +} + +// createRootfs creates a minimal root filesystem below the specified +// directory. +func createRootfs(dir string) error { + err := os.MkdirAll(dir, testDirMode) + if err != nil { + return err + } + + container, err := RunCommand([]string{"docker", "create", testDockerImage}) + if err != nil { + return err + } + + cmd1 := exec.Command("docker", "export", container) + cmd2 := exec.Command("tar", "-C", dir, "-xvf", "-") + + cmd1Stdout, err := cmd1.StdoutPipe() + if err != nil { + return err + } + + cmd2.Stdin = cmd1Stdout + + err = cmd2.Start() + if err != nil { + return err + } + + err = cmd1.Run() + if err != nil { + return err + } + + err = cmd2.Wait() + if err != nil { + return err + } + + // Clean up + _, err = RunCommand([]string{"docker", "rm", container}) + if err != nil { + return err + } + + return nil } func createFile(file, contents string) error { @@ -211,3 +366,17 @@ func TestGetFileContents(t *testing.T) { assert.Equal(t, contents, d.contents) } } + +func TestIsEphemeralStorage(t *testing.T) { + sampleEphePath := "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/kubernetes.io~empty-dir/cache-volume" + isEphe := IsEphemeralStorage(sampleEphePath) + if !isEphe { + t.Fatalf("Unable to correctly determine volume type") + } + + sampleEphePath = "/var/lib/kubelet/pods/366c3a75-4869-11e8-b479-507b9ddd5ce4/volumes/cache-volume" + isEphe = IsEphemeralStorage(sampleEphePath) + if isEphe { + t.Fatalf("Unable to correctly determine volume type") + } +}