From 566f411b081c078b0caf9bfb32cde244e7c5c091 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Thu, 31 Aug 2017 14:17:34 -0400 Subject: [PATCH] Support remote runtimes with native cAdvisor support --- cmd/kubelet/app/server.go | 3 +- pkg/kubelet/cadvisor/BUILD | 2 +- pkg/kubelet/cadvisor/cadvisor_linux.go | 26 ++++++---------- pkg/kubelet/cadvisor/cadvisor_unsupported.go | 2 +- pkg/kubelet/cadvisor/cadvisor_windows.go | 2 +- pkg/kubelet/cadvisor/types.go | 8 ++++- pkg/kubelet/cadvisor/util.go | 32 ++++++++++++++++++++ test/e2e_node/environment/conformance.go | 2 +- 8 files changed, 54 insertions(+), 23 deletions(-) diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index e700c89de0c..0f30c2304a5 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -392,7 +392,8 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) { } if kubeDeps.CAdvisorInterface == nil { - kubeDeps.CAdvisorInterface, err = cadvisor.New(s.Address, uint(s.CAdvisorPort), s.ContainerRuntime, s.RootDirectory) + imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntime, s.RemoteRuntimeEndpoint) + kubeDeps.CAdvisorInterface, err = cadvisor.New(s.Address, uint(s.CAdvisorPort), imageFsInfoProvider, s.RootDirectory) if err != nil { return err } diff --git a/pkg/kubelet/cadvisor/BUILD b/pkg/kubelet/cadvisor/BUILD index 15cea0e8d72..41aa63a58e7 100644 --- a/pkg/kubelet/cadvisor/BUILD +++ b/pkg/kubelet/cadvisor/BUILD @@ -24,6 +24,7 @@ go_library( }), deps = [ "//vendor/github.com/google/cadvisor/events:go_default_library", + "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", @@ -34,7 +35,6 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/cache/memory:go_default_library", "//vendor/github.com/google/cadvisor/container:go_default_library", - "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/http:go_default_library", "//vendor/github.com/google/cadvisor/manager:go_default_library", "//vendor/github.com/google/cadvisor/metrics:go_default_library", diff --git a/pkg/kubelet/cadvisor/cadvisor_linux.go b/pkg/kubelet/cadvisor/cadvisor_linux.go index bcde373dcfc..4749016a49a 100644 --- a/pkg/kubelet/cadvisor/cadvisor_linux.go +++ b/pkg/kubelet/cadvisor/cadvisor_linux.go @@ -32,7 +32,6 @@ import ( "github.com/google/cadvisor/cache/memory" cadvisormetrics "github.com/google/cadvisor/container" "github.com/google/cadvisor/events" - cadvisorfs "github.com/google/cadvisor/fs" cadvisorhttp "github.com/google/cadvisor/http" cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" @@ -44,8 +43,8 @@ import ( ) type cadvisorClient struct { - runtime string - rootPath string + imageFsInfoProvider ImageFsInfoProvider + rootPath string manager.Manager } @@ -106,7 +105,7 @@ func containerLabels(c *cadvisorapi.ContainerInfo) map[string]string { } // New creates a cAdvisor and exports its API on the specified port if port > 0. -func New(address string, port uint, runtime string, rootPath string) (Interface, error) { +func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string) (Interface, error) { sysFs := sysfs.NewRealSysFs() // Create and start the cAdvisor container manager. @@ -126,9 +125,9 @@ func New(address string, port uint, runtime string, rootPath string) (Interface, } cadvisorClient := &cadvisorClient{ - runtime: runtime, - rootPath: rootPath, - Manager: m, + imageFsInfoProvider: imageFsInfoProvider, + rootPath: rootPath, + Manager: m, } err = cadvisorClient.exportHTTP(address, port) @@ -208,17 +207,10 @@ func (cc *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) { } func (cc *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { - var label string - - switch cc.runtime { - case "docker": - label = cadvisorfs.LabelDockerImages - case "rkt": - label = cadvisorfs.LabelRktImages - default: - return cadvisorapiv2.FsInfo{}, fmt.Errorf("ImagesFsInfo: unknown runtime: %v", cc.runtime) + label, err := cc.imageFsInfoProvider.ImageFsInfoLabel() + if err != nil { + return cadvisorapiv2.FsInfo{}, err } - return cc.getFsInfo(label) } diff --git a/pkg/kubelet/cadvisor/cadvisor_unsupported.go b/pkg/kubelet/cadvisor/cadvisor_unsupported.go index 788629c3392..ef21e8c86ec 100644 --- a/pkg/kubelet/cadvisor/cadvisor_unsupported.go +++ b/pkg/kubelet/cadvisor/cadvisor_unsupported.go @@ -31,7 +31,7 @@ type cadvisorUnsupported struct { var _ Interface = new(cadvisorUnsupported) -func New(address string, port uint, runtime string, rootPath string) (Interface, error) { +func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string) (Interface, error) { return &cadvisorUnsupported{}, nil } diff --git a/pkg/kubelet/cadvisor/cadvisor_windows.go b/pkg/kubelet/cadvisor/cadvisor_windows.go index f7e265ec678..ec0764293b3 100644 --- a/pkg/kubelet/cadvisor/cadvisor_windows.go +++ b/pkg/kubelet/cadvisor/cadvisor_windows.go @@ -30,7 +30,7 @@ type cadvisorClient struct { var _ Interface = new(cadvisorClient) // New creates a cAdvisor and exports its API on the specified port if port > 0. -func New(address string, port uint, runtime string, rootPath string) (Interface, error) { +func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string) (Interface, error) { return &cadvisorClient{}, nil } diff --git a/pkg/kubelet/cadvisor/types.go b/pkg/kubelet/cadvisor/types.go index 2a97ba35237..9ad6267ec21 100644 --- a/pkg/kubelet/cadvisor/types.go +++ b/pkg/kubelet/cadvisor/types.go @@ -33,7 +33,7 @@ type Interface interface { VersionInfo() (*cadvisorapi.VersionInfo, error) - // Returns usage information about the filesystem holding Docker images. + // Returns usage information about the filesystem holding container images. ImagesFsInfo() (cadvisorapiv2.FsInfo, error) // Returns usage information about the root filesystem. @@ -45,3 +45,9 @@ type Interface interface { // HasDedicatedImageFs returns true iff a dedicated image filesystem exists for storing images. HasDedicatedImageFs() (bool, error) } + +// ImageFsInfoProvider informs cAdvisor how to find imagefs for container images. +type ImageFsInfoProvider interface { + // ImageFsInfoLabel returns the label cAdvisor should use to find the filesystem holding container images. + ImageFsInfoLabel() (string, error) +} diff --git a/pkg/kubelet/cadvisor/util.go b/pkg/kubelet/cadvisor/util.go index a95a8fb19e2..a2a6e0b96e8 100644 --- a/pkg/kubelet/cadvisor/util.go +++ b/pkg/kubelet/cadvisor/util.go @@ -17,12 +17,44 @@ limitations under the License. package cadvisor import ( + "fmt" + + cadvisorfs "github.com/google/cadvisor/fs" cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapi2 "github.com/google/cadvisor/info/v2" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) +// imageFsInfoProvider knows how to translate the configured runtime +// to its file system label for images. +type imageFsInfoProvider struct { + runtime string + runtimeEndpoint string +} + +// ImageFsInfoLabel returns the image fs label for the configured runtime. +// For remote runtimes, it handles additional runtimes natively understood by cAdvisor. +func (i *imageFsInfoProvider) ImageFsInfoLabel() (string, error) { + switch i.runtime { + case "docker": + return cadvisorfs.LabelDockerImages, nil + case "rkt": + return cadvisorfs.LabelRktImages, nil + case "remote": + // TODO: pending rebase including https://github.com/google/cadvisor/pull/1741 + if i.runtimeEndpoint == "/var/run/crio.sock" { + return "crio-images", nil + } + } + return "", fmt.Errorf("no imagefs label for configured runtime") +} + +// NewImageFsInfoProvider returns a provider for the specified runtime configuration. +func NewImageFsInfoProvider(runtime, runtimeEndpoint string) ImageFsInfoProvider { + return &imageFsInfoProvider{runtime: runtime, runtimeEndpoint: runtimeEndpoint} +} + func CapacityFromMachineInfo(info *cadvisorapi.MachineInfo) v1.ResourceList { c := v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity( diff --git a/test/e2e_node/environment/conformance.go b/test/e2e_node/environment/conformance.go index d9dac60fe71..0b887c07bd5 100644 --- a/test/e2e_node/environment/conformance.go +++ b/test/e2e_node/environment/conformance.go @@ -99,7 +99,7 @@ func containerRuntime() error { } // Setup cadvisor to check the container environment - c, err := cadvisor.New("", 0 /*don't start the http server*/, "docker", "/var/lib/kubelet") + c, err := cadvisor.New("", 0 /*don't start the http server*/, cadvisor.NewImageFsInfoProvider("docker", ""), "/var/lib/kubelet") if err != nil { return printError("Container Runtime Check: %s Could not start cadvisor %v", failed, err) }