From 569bd780f143306450fcc684a23753f7a8dcd9e8 Mon Sep 17 00:00:00 2001 From: Wang Liang Date: Fri, 11 Oct 2019 23:04:32 -0400 Subject: [PATCH 1/3] virtcontainers: change pass by value to pass by reference container.config does not point to sandbox.config.Containers.ContainerConfig which caused the ContainerConfig not sync. Fixes: #2129 Signed-off-by: Wang Liang --- virtcontainers/container.go | 6 +++--- virtcontainers/sandbox.go | 6 +++--- virtcontainers/sandbox_test.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/virtcontainers/container.go b/virtcontainers/container.go index be4f043ced..aff4e85696 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -717,7 +717,7 @@ func (c *Container) createBlockDevices() error { } // newContainer creates a Container structure from a sandbox and a container configuration. -func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, error) { +func newContainer(sandbox *Sandbox, contConfig *ContainerConfig) (*Container, error) { span, _ := sandbox.trace("newContainer") defer span.Finish() @@ -729,7 +729,7 @@ func newContainer(sandbox *Sandbox, contConfig ContainerConfig) (*Container, err id: contConfig.ID, sandboxID: sandbox.id, rootFs: contConfig.RootFs, - config: &contConfig, + config: contConfig, sandbox: sandbox, runPath: store.ContainerRuntimeRootPath(sandbox.id, contConfig.ID), configPath: store.ContainerConfigurationRootPath(sandbox.id, contConfig.ID), @@ -812,7 +812,7 @@ func (c *Container) createMounts() error { return nil } -func (c *Container) createDevices(contConfig ContainerConfig) error { +func (c *Container) createDevices(contConfig *ContainerConfig) error { // If sandbox supports "newstore", only newly created container can reach this function, // so we don't call restore when `supportNewStore` is true if !c.sandbox.supportNewStore() { diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 04b6f5029a..6458e69812 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -1099,7 +1099,7 @@ func (s *Sandbox) fetchContainers() error { contConfig.Spec = &spec s.config.Containers[i] = contConfig - c, err := newContainer(s, contConfig) + c, err := newContainer(s, &s.config.Containers[i]) if err != nil { return err } @@ -1118,7 +1118,7 @@ func (s *Sandbox) fetchContainers() error { func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, error) { storeAlreadyExists := store.VCContainerStoreExists(s.ctx, s.id, contConfig.ID) // Create the container. - c, err := newContainer(s, contConfig) + c, err := newContainer(s, &contConfig) if err != nil { return nil, err } @@ -1417,7 +1417,7 @@ func (s *Sandbox) createContainers() error { for _, contConfig := range s.config.Containers { - c, err := newContainer(s, contConfig) + c, err := newContainer(s, &contConfig) if err != nil { return err } diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go index 807f3e9104..e6d38a310b 100644 --- a/virtcontainers/sandbox_test.go +++ b/virtcontainers/sandbox_test.go @@ -621,7 +621,7 @@ func TestSandboxGetContainer(t *testing.T) { contID := "999" contConfig := newTestContainerConfigNoop(contID) - nc, err := newContainer(p, contConfig) + nc, err := newContainer(p, &contConfig) assert.NoError(err) err = p.addContainer(nc) @@ -1043,7 +1043,7 @@ func TestDeleteStoreWhenNewContainerFail(t *testing.T) { DevType: "", }, } - _, err = newContainer(p, contConfig) + _, err = newContainer(p, &contConfig) assert.NotNil(t, err, "New container with invalid device info should fail") storePath := store.ContainerConfigurationRootPath(testSandboxID, contID) _, err = os.Stat(storePath) From 1bbc1d58bd7ec3ca712a619ee6dcc5262250b9ee Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Tue, 17 Sep 2019 14:47:30 -0500 Subject: [PATCH 2/3] virtcontainers: add StatsSandbox to vc API StatsSandbox is used to gather metrics for the sandbox (host cgroup) as well as from the individual containers (from the guest cgroups). This is intended to be used for easily calculating Kata sandbox overheads. Fixes: #2096 Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Eric Ernst --- virtcontainers/api.go | 40 +++++++++++++++++++++++++ virtcontainers/implementation.go | 5 ++++ virtcontainers/interfaces.go | 1 + virtcontainers/pkg/vcmock/mock.go | 9 ++++++ virtcontainers/pkg/vcmock/types.go | 1 + virtcontainers/sandbox.go | 47 +++++++++++++++++++++++++++++- 6 files changed, 102 insertions(+), 1 deletion(-) diff --git a/virtcontainers/api.go b/virtcontainers/api.go index 7541823ad7..b28e6ecab9 100644 --- a/virtcontainers/api.go +++ b/virtcontainers/api.go @@ -758,6 +758,46 @@ func StatsContainer(ctx context.Context, sandboxID, containerID string) (Contain return s.StatsContainer(containerID) } +// StatsSandbox is the virtcontainers sandbox stats entry point. +// StatsSandbox returns a detailed sandbox stats. +func StatsSandbox(ctx context.Context, sandboxID string) (SandboxStats, []ContainerStats, error) { + span, ctx := trace(ctx, "StatsSandbox") + defer span.Finish() + + if sandboxID == "" { + return SandboxStats{}, []ContainerStats{}, vcTypes.ErrNeedSandboxID + } + + lockFile, err := rLockSandbox(ctx, sandboxID) + if err != nil { + return SandboxStats{}, []ContainerStats{}, err + } + + defer unlockSandbox(ctx, sandboxID, lockFile) + + s, err := fetchSandbox(ctx, sandboxID) + if err != nil { + return SandboxStats{}, []ContainerStats{}, err + } + defer s.releaseStatelessSandbox() + + sandboxStats, err := s.Stats() + if err != nil { + return SandboxStats{}, []ContainerStats{}, err + } + + containerStats := []ContainerStats{} + for _, c := range s.containers { + cstats, err := s.StatsContainer(c.id) + if err != nil { + return SandboxStats{}, []ContainerStats{}, err + } + containerStats = append(containerStats, cstats) + } + + return sandboxStats, containerStats, nil +} + func togglePauseContainer(ctx context.Context, sandboxID, containerID string, pause bool) error { if sandboxID == "" { return vcTypes.ErrNeedSandboxID diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go index e59bcc257e..4bc2dbead6 100644 --- a/virtcontainers/implementation.go +++ b/virtcontainers/implementation.go @@ -122,6 +122,11 @@ func (impl *VCImpl) StatsContainer(ctx context.Context, sandboxID, containerID s return StatsContainer(ctx, sandboxID, containerID) } +// StatsSandbox implements the VC function of the same name. +func (impl *VCImpl) StatsSandbox(ctx context.Context, sandboxID string) (SandboxStats, []ContainerStats, error) { + return StatsSandbox(ctx, sandboxID) +} + // KillContainer implements the VC function of the same name. func (impl *VCImpl) KillContainer(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error { return KillContainer(ctx, sandboxID, containerID, signal, all) diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go index 6f73ca8ad2..84ee0cb0a9 100644 --- a/virtcontainers/interfaces.go +++ b/virtcontainers/interfaces.go @@ -41,6 +41,7 @@ type VC interface { StartContainer(ctx context.Context, sandboxID, containerID string) (VCContainer, error) StatusContainer(ctx context.Context, sandboxID, containerID string) (ContainerStatus, error) StatsContainer(ctx context.Context, sandboxID, containerID string) (ContainerStats, error) + StatsSandbox(ctx context.Context, sandboxID string) (SandboxStats, []ContainerStats, error) StopContainer(ctx context.Context, sandboxID, containerID string) (VCContainer, error) ProcessListContainer(ctx context.Context, sandboxID, containerID string, options ProcessListOptions) (ProcessList, error) UpdateContainer(ctx context.Context, sandboxID, containerID string, resources specs.LinuxResources) error diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go index 3af4f2ead2..b09e5bd78f 100644 --- a/virtcontainers/pkg/vcmock/mock.go +++ b/virtcontainers/pkg/vcmock/mock.go @@ -200,6 +200,15 @@ func (m *VCMock) StatsContainer(ctx context.Context, sandboxID, containerID stri return vc.ContainerStats{}, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) } +// StatsSandbox implements the VC function of the same name. +func (m *VCMock) StatsSandbox(ctx context.Context, sandboxID string) (vc.SandboxStats, []vc.ContainerStats, error) { + if m.StatsContainerFunc != nil { + return m.StatsSandboxFunc(ctx, sandboxID) + } + + return vc.SandboxStats{}, []vc.ContainerStats{}, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) +} + // KillContainer implements the VC function of the same name. func (m *VCMock) KillContainer(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error { if m.KillContainerFunc != nil { diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go index f141993ec3..51d7111012 100644 --- a/virtcontainers/pkg/vcmock/types.go +++ b/virtcontainers/pkg/vcmock/types.go @@ -54,6 +54,7 @@ type VCMock struct { StartSandboxFunc func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) StatusSandboxFunc func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) StatsContainerFunc func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStats, error) + StatsSandboxFunc func(ctx context.Context, sandboxID string) (vc.SandboxStats, []vc.ContainerStats, error) StopSandboxFunc func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) CreateContainerFunc func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 6458e69812..1fddb4f4a4 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -63,6 +63,12 @@ type SandboxStatus struct { Annotations map[string]string } +// SandboxStats describes a sandbox's stats +type SandboxStats struct { + CgroupStats CgroupStats + Cpus int +} + // SandboxConfig is a Sandbox configuration. type SandboxConfig struct { ID string @@ -1367,6 +1373,46 @@ func (s *Sandbox) StatsContainer(containerID string) (ContainerStats, error) { return *stats, nil } +// Stats returns the stats of a running sandbox +func (s *Sandbox) Stats() (SandboxStats, error) { + if s.state.CgroupPath == "" { + return SandboxStats{}, fmt.Errorf("sandbox cgroup path is emtpy") + } + + var path string + var cgroupSubsystems cgroups.Hierarchy + + if s.config.SandboxCgroupOnly { + cgroupSubsystems = cgroups.V1 + path = s.state.CgroupPath + } else { + cgroupSubsystems = V1NoConstraints + path = cgroupNoConstraintsPath(s.state.CgroupPath) + } + + cgroup, err := cgroupsLoadFunc(cgroupSubsystems, cgroups.StaticPath(path)) + if err != nil { + return SandboxStats{}, fmt.Errorf("Could not load sandbox cgroup in %v: %v", s.state.CgroupPath, err) + } + + metrics, err := cgroup.Stat(cgroups.ErrorHandler(cgroups.IgnoreNotExist)) + if err != nil { + return SandboxStats{}, err + } + + stats := SandboxStats{} + + stats.CgroupStats.CPUStats.CPUUsage.TotalUsage = metrics.CPU.Usage.Total + stats.CgroupStats.MemoryStats.Usage.Usage = metrics.Memory.Usage.Usage + tids, err := s.hypervisor.getThreadIDs() + if err != nil { + return stats, err + } + stats.Cpus = len(tids.vcpus) + + return stats, nil +} + // PauseContainer pauses a running container. func (s *Sandbox) PauseContainer(containerID string) error { // Fetch the container. @@ -2125,7 +2171,6 @@ func (s *Sandbox) setupSandboxCgroup() error { if err := cgroup.Add(cgroups.Process{Pid: runtimePid}); err != nil { return fmt.Errorf("Could not add runtime PID %d to sandbox cgroup: %v", runtimePid, err) } - return nil } From 7fe01004442f77bcbfd6455477120046760ddb46 Mon Sep 17 00:00:00 2001 From: Jose Carlos Venegas Munoz Date: Tue, 17 Sep 2019 14:47:30 -0500 Subject: [PATCH 3/3] cli: add kata-overhead command Introduce kata-overhead command to kata-runtime CLI, to help with calculating sandbox overhead. Fixes: #2096 Signed-off-by: Jose Carlos Venegas Munoz Signed-off-by: Eric Ernst --- cli/kata-overhead.go | 136 +++++++++++++++++++++++++++++++++++++++++++ cli/main.go | 1 + 2 files changed, 137 insertions(+) create mode 100644 cli/kata-overhead.go diff --git a/cli/kata-overhead.go b/cli/kata-overhead.go new file mode 100644 index 0000000000..8214152152 --- /dev/null +++ b/cli/kata-overhead.go @@ -0,0 +1,136 @@ +// +build cgo,linux +// Copyright (c) 2014,2015,2016 Docker, Inc. +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "context" + "fmt" + "time" + + "github.com/kata-containers/runtime/pkg/katautils" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var kataOverheadCLICommand = cli.Command{ + Name: "kata-overhead", + Usage: "provides kata overhead at sandbox level", + ArgsUsage: ` [sandbox-id...] + + is your name for the instance of the sandbox.`, + + Description: `The kata-overhead command shows the overhead of a running Kata sandbox. Overhead + is calculated as the sum of pod resource utilization as measured on host cgroup minus the total + container usage measured inside the Kata guest for each container's cgroup.`, + + Action: func(context *cli.Context) error { + ctx, err := cliContextToContext(context) + if err != nil { + return err + } + + args := context.Args() + if !args.Present() { + return fmt.Errorf("Missing container ID, should at least provide one") + } + + for _, cID := range []string(args) { + if err := overhead(ctx, cID); err != nil { + return err + } + } + + return nil + }, +} + +func overhead(ctx context.Context, containerID string) error { + span, _ := katautils.Trace(ctx, "overhead") + defer span.Finish() + + kataLog = kataLog.WithField("container", containerID) + setExternalLoggers(ctx, kataLog) + span.SetTag("container", containerID) + + status, sandboxID, err := getExistingContainerInfo(ctx, containerID) + if err != nil { + return err + } + + containerID = status.ID + + kataLog = kataLog.WithFields(logrus.Fields{ + "container": containerID, + "sandbox": sandboxID, + }) + + setExternalLoggers(ctx, kataLog) + span.SetTag("container", containerID) + span.SetTag("sandbox", sandboxID) + + if status.State.State == types.StateStopped { + return fmt.Errorf("container with id %s is not running", status.ID) + } + + initTime := time.Now().UnixNano() + + initialSandboxStats, initialContainerStats, err := vci.StatsSandbox(ctx, sandboxID) + if err != nil { + return err + } + + hostInitCPU := initialSandboxStats.CgroupStats.CPUStats.CPUUsage.TotalUsage + guestInitCPU := uint64(0) + for _, cs := range initialContainerStats { + guestInitCPU += cs.CgroupStats.CPUStats.CPUUsage.TotalUsage + } + + // Wait for 1 second to calculate CPU usage + time.Sleep(time.Second * 1) + finishtTime := time.Now().UnixNano() + + finishSandboxStats, finishContainersStats, err := vci.StatsSandbox(ctx, sandboxID) + if err != nil { + return err + } + + hostFinalCPU := finishSandboxStats.CgroupStats.CPUStats.CPUUsage.TotalUsage + guestFinalCPU := uint64(0) + for _, cs := range finishContainersStats { + guestFinalCPU += cs.CgroupStats.CPUStats.CPUUsage.TotalUsage + } + + var guestMemoryUsage uint64 + for _, cs := range finishContainersStats { + guestMemoryUsage += cs.CgroupStats.MemoryStats.Usage.Usage + } + + hostMemoryUsage := finishSandboxStats.CgroupStats.MemoryStats.Usage.Usage + deltaTime := finishtTime - initTime + + cpuUsageGuest := float64(guestFinalCPU-guestInitCPU) / float64(deltaTime) * 100 + cpuUsageHost := float64(hostFinalCPU-hostInitCPU) / float64(deltaTime) * 100 + + fmt.Printf("Sandbox overhead for container: %s\n", containerID) + fmt.Printf("cpu_overhead=%f\n", cpuUsageHost-cpuUsageGuest) + fmt.Printf("memory_overhead_bytes=%d\n\n", hostMemoryUsage-guestMemoryUsage) + fmt.Printf(" --CPU details--\n") + fmt.Printf("cpu_host=%f\n", cpuUsageHost) + fmt.Printf("\tcpu_host_init=%d\n", hostInitCPU) + fmt.Printf("\tcpu_host_final=%d\n", hostFinalCPU) + fmt.Printf("cpu_guest=%f\n", cpuUsageGuest) + fmt.Printf("\tcpu_guest_init=%d\n", guestInitCPU) + fmt.Printf("\tcpu_guest_final=%d\n", guestFinalCPU) + fmt.Printf("Number of available vCPUs=%d\n", finishSandboxStats.Cpus) + fmt.Printf(" --Memory details--\n") + fmt.Printf("memory_host_bytes=%d\n", hostMemoryUsage) + fmt.Printf("memory_guest_bytes=%d\n\n", guestMemoryUsage) + + return nil +} diff --git a/cli/main.go b/cli/main.go index ef220196ad..8a624a4c8c 100644 --- a/cli/main.go +++ b/cli/main.go @@ -134,6 +134,7 @@ var runtimeCommands = []cli.Command{ kataCheckCLICommand, kataEnvCLICommand, kataNetworkCLICommand, + kataOverheadCLICommand, factoryCLICommand, }