diff --git a/cli/kata-overhead.go b/cli/kata-overhead.go new file mode 100644 index 0000000000..8214152152 --- /dev/null +++ b/cli/kata-overhead.go @@ -0,0 +1,136 @@ +// +build cgo,linux +// Copyright (c) 2014,2015,2016 Docker, Inc. +// Copyright (c) 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "context" + "fmt" + "time" + + "github.com/kata-containers/runtime/pkg/katautils" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var kataOverheadCLICommand = cli.Command{ + Name: "kata-overhead", + Usage: "provides kata overhead at sandbox level", + ArgsUsage: ` [sandbox-id...] + + is your name for the instance of the sandbox.`, + + Description: `The kata-overhead command shows the overhead of a running Kata sandbox. Overhead + is calculated as the sum of pod resource utilization as measured on host cgroup minus the total + container usage measured inside the Kata guest for each container's cgroup.`, + + Action: func(context *cli.Context) error { + ctx, err := cliContextToContext(context) + if err != nil { + return err + } + + args := context.Args() + if !args.Present() { + return fmt.Errorf("Missing container ID, should at least provide one") + } + + for _, cID := range []string(args) { + if err := overhead(ctx, cID); err != nil { + return err + } + } + + return nil + }, +} + +func overhead(ctx context.Context, containerID string) error { + span, _ := katautils.Trace(ctx, "overhead") + defer span.Finish() + + kataLog = kataLog.WithField("container", containerID) + setExternalLoggers(ctx, kataLog) + span.SetTag("container", containerID) + + status, sandboxID, err := getExistingContainerInfo(ctx, containerID) + if err != nil { + return err + } + + containerID = status.ID + + kataLog = kataLog.WithFields(logrus.Fields{ + "container": containerID, + "sandbox": sandboxID, + }) + + setExternalLoggers(ctx, kataLog) + span.SetTag("container", containerID) + span.SetTag("sandbox", sandboxID) + + if status.State.State == types.StateStopped { + return fmt.Errorf("container with id %s is not running", status.ID) + } + + initTime := time.Now().UnixNano() + + initialSandboxStats, initialContainerStats, err := vci.StatsSandbox(ctx, sandboxID) + if err != nil { + return err + } + + hostInitCPU := initialSandboxStats.CgroupStats.CPUStats.CPUUsage.TotalUsage + guestInitCPU := uint64(0) + for _, cs := range initialContainerStats { + guestInitCPU += cs.CgroupStats.CPUStats.CPUUsage.TotalUsage + } + + // Wait for 1 second to calculate CPU usage + time.Sleep(time.Second * 1) + finishtTime := time.Now().UnixNano() + + finishSandboxStats, finishContainersStats, err := vci.StatsSandbox(ctx, sandboxID) + if err != nil { + return err + } + + hostFinalCPU := finishSandboxStats.CgroupStats.CPUStats.CPUUsage.TotalUsage + guestFinalCPU := uint64(0) + for _, cs := range finishContainersStats { + guestFinalCPU += cs.CgroupStats.CPUStats.CPUUsage.TotalUsage + } + + var guestMemoryUsage uint64 + for _, cs := range finishContainersStats { + guestMemoryUsage += cs.CgroupStats.MemoryStats.Usage.Usage + } + + hostMemoryUsage := finishSandboxStats.CgroupStats.MemoryStats.Usage.Usage + deltaTime := finishtTime - initTime + + cpuUsageGuest := float64(guestFinalCPU-guestInitCPU) / float64(deltaTime) * 100 + cpuUsageHost := float64(hostFinalCPU-hostInitCPU) / float64(deltaTime) * 100 + + fmt.Printf("Sandbox overhead for container: %s\n", containerID) + fmt.Printf("cpu_overhead=%f\n", cpuUsageHost-cpuUsageGuest) + fmt.Printf("memory_overhead_bytes=%d\n\n", hostMemoryUsage-guestMemoryUsage) + fmt.Printf(" --CPU details--\n") + fmt.Printf("cpu_host=%f\n", cpuUsageHost) + fmt.Printf("\tcpu_host_init=%d\n", hostInitCPU) + fmt.Printf("\tcpu_host_final=%d\n", hostFinalCPU) + fmt.Printf("cpu_guest=%f\n", cpuUsageGuest) + fmt.Printf("\tcpu_guest_init=%d\n", guestInitCPU) + fmt.Printf("\tcpu_guest_final=%d\n", guestFinalCPU) + fmt.Printf("Number of available vCPUs=%d\n", finishSandboxStats.Cpus) + fmt.Printf(" --Memory details--\n") + fmt.Printf("memory_host_bytes=%d\n", hostMemoryUsage) + fmt.Printf("memory_guest_bytes=%d\n\n", guestMemoryUsage) + + return nil +} diff --git a/cli/main.go b/cli/main.go index ef220196ad..8a624a4c8c 100644 --- a/cli/main.go +++ b/cli/main.go @@ -134,6 +134,7 @@ var runtimeCommands = []cli.Command{ kataCheckCLICommand, kataEnvCLICommand, kataNetworkCLICommand, + kataOverheadCLICommand, factoryCLICommand, } diff --git a/virtcontainers/api.go b/virtcontainers/api.go index 7541823ad7..b28e6ecab9 100644 --- a/virtcontainers/api.go +++ b/virtcontainers/api.go @@ -758,6 +758,46 @@ func StatsContainer(ctx context.Context, sandboxID, containerID string) (Contain return s.StatsContainer(containerID) } +// StatsSandbox is the virtcontainers sandbox stats entry point. +// StatsSandbox returns a detailed sandbox stats. +func StatsSandbox(ctx context.Context, sandboxID string) (SandboxStats, []ContainerStats, error) { + span, ctx := trace(ctx, "StatsSandbox") + defer span.Finish() + + if sandboxID == "" { + return SandboxStats{}, []ContainerStats{}, vcTypes.ErrNeedSandboxID + } + + lockFile, err := rLockSandbox(ctx, sandboxID) + if err != nil { + return SandboxStats{}, []ContainerStats{}, err + } + + defer unlockSandbox(ctx, sandboxID, lockFile) + + s, err := fetchSandbox(ctx, sandboxID) + if err != nil { + return SandboxStats{}, []ContainerStats{}, err + } + defer s.releaseStatelessSandbox() + + sandboxStats, err := s.Stats() + if err != nil { + return SandboxStats{}, []ContainerStats{}, err + } + + containerStats := []ContainerStats{} + for _, c := range s.containers { + cstats, err := s.StatsContainer(c.id) + if err != nil { + return SandboxStats{}, []ContainerStats{}, err + } + containerStats = append(containerStats, cstats) + } + + return sandboxStats, containerStats, nil +} + func togglePauseContainer(ctx context.Context, sandboxID, containerID string, pause bool) error { if sandboxID == "" { return vcTypes.ErrNeedSandboxID diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go index e59bcc257e..4bc2dbead6 100644 --- a/virtcontainers/implementation.go +++ b/virtcontainers/implementation.go @@ -122,6 +122,11 @@ func (impl *VCImpl) StatsContainer(ctx context.Context, sandboxID, containerID s return StatsContainer(ctx, sandboxID, containerID) } +// StatsSandbox implements the VC function of the same name. +func (impl *VCImpl) StatsSandbox(ctx context.Context, sandboxID string) (SandboxStats, []ContainerStats, error) { + return StatsSandbox(ctx, sandboxID) +} + // KillContainer implements the VC function of the same name. func (impl *VCImpl) KillContainer(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error { return KillContainer(ctx, sandboxID, containerID, signal, all) diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go index 6f73ca8ad2..84ee0cb0a9 100644 --- a/virtcontainers/interfaces.go +++ b/virtcontainers/interfaces.go @@ -41,6 +41,7 @@ type VC interface { StartContainer(ctx context.Context, sandboxID, containerID string) (VCContainer, error) StatusContainer(ctx context.Context, sandboxID, containerID string) (ContainerStatus, error) StatsContainer(ctx context.Context, sandboxID, containerID string) (ContainerStats, error) + StatsSandbox(ctx context.Context, sandboxID string) (SandboxStats, []ContainerStats, error) StopContainer(ctx context.Context, sandboxID, containerID string) (VCContainer, error) ProcessListContainer(ctx context.Context, sandboxID, containerID string, options ProcessListOptions) (ProcessList, error) UpdateContainer(ctx context.Context, sandboxID, containerID string, resources specs.LinuxResources) error diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go index 3af4f2ead2..b09e5bd78f 100644 --- a/virtcontainers/pkg/vcmock/mock.go +++ b/virtcontainers/pkg/vcmock/mock.go @@ -200,6 +200,15 @@ func (m *VCMock) StatsContainer(ctx context.Context, sandboxID, containerID stri return vc.ContainerStats{}, fmt.Errorf("%s: %s (%+v): sandboxID: %v, containerID: %v", mockErrorPrefix, getSelf(), m, sandboxID, containerID) } +// StatsSandbox implements the VC function of the same name. +func (m *VCMock) StatsSandbox(ctx context.Context, sandboxID string) (vc.SandboxStats, []vc.ContainerStats, error) { + if m.StatsContainerFunc != nil { + return m.StatsSandboxFunc(ctx, sandboxID) + } + + return vc.SandboxStats{}, []vc.ContainerStats{}, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) +} + // KillContainer implements the VC function of the same name. func (m *VCMock) KillContainer(ctx context.Context, sandboxID, containerID string, signal syscall.Signal, all bool) error { if m.KillContainerFunc != nil { diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go index f141993ec3..51d7111012 100644 --- a/virtcontainers/pkg/vcmock/types.go +++ b/virtcontainers/pkg/vcmock/types.go @@ -54,6 +54,7 @@ type VCMock struct { StartSandboxFunc func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) StatusSandboxFunc func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) StatsContainerFunc func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStats, error) + StatsSandboxFunc func(ctx context.Context, sandboxID string) (vc.SandboxStats, []vc.ContainerStats, error) StopSandboxFunc func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) CreateContainerFunc func(ctx context.Context, sandboxID string, containerConfig vc.ContainerConfig) (vc.VCSandbox, vc.VCContainer, error) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 6458e69812..1fddb4f4a4 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -63,6 +63,12 @@ type SandboxStatus struct { Annotations map[string]string } +// SandboxStats describes a sandbox's stats +type SandboxStats struct { + CgroupStats CgroupStats + Cpus int +} + // SandboxConfig is a Sandbox configuration. type SandboxConfig struct { ID string @@ -1367,6 +1373,46 @@ func (s *Sandbox) StatsContainer(containerID string) (ContainerStats, error) { return *stats, nil } +// Stats returns the stats of a running sandbox +func (s *Sandbox) Stats() (SandboxStats, error) { + if s.state.CgroupPath == "" { + return SandboxStats{}, fmt.Errorf("sandbox cgroup path is emtpy") + } + + var path string + var cgroupSubsystems cgroups.Hierarchy + + if s.config.SandboxCgroupOnly { + cgroupSubsystems = cgroups.V1 + path = s.state.CgroupPath + } else { + cgroupSubsystems = V1NoConstraints + path = cgroupNoConstraintsPath(s.state.CgroupPath) + } + + cgroup, err := cgroupsLoadFunc(cgroupSubsystems, cgroups.StaticPath(path)) + if err != nil { + return SandboxStats{}, fmt.Errorf("Could not load sandbox cgroup in %v: %v", s.state.CgroupPath, err) + } + + metrics, err := cgroup.Stat(cgroups.ErrorHandler(cgroups.IgnoreNotExist)) + if err != nil { + return SandboxStats{}, err + } + + stats := SandboxStats{} + + stats.CgroupStats.CPUStats.CPUUsage.TotalUsage = metrics.CPU.Usage.Total + stats.CgroupStats.MemoryStats.Usage.Usage = metrics.Memory.Usage.Usage + tids, err := s.hypervisor.getThreadIDs() + if err != nil { + return stats, err + } + stats.Cpus = len(tids.vcpus) + + return stats, nil +} + // PauseContainer pauses a running container. func (s *Sandbox) PauseContainer(containerID string) error { // Fetch the container. @@ -2125,7 +2171,6 @@ func (s *Sandbox) setupSandboxCgroup() error { if err := cgroup.Add(cgroups.Process{Pid: runtimePid}); err != nil { return fmt.Errorf("Could not add runtime PID %d to sandbox cgroup: %v", runtimePid, err) } - return nil }