Merge pull request #51728 from derekwaynecarr/cadvisor-stats

Automatic merge from submit-queue (batch tested with PRs 51728, 49202)

Enable CRI-O stats from cAdvisor

**What this PR does / why we need it**:
cAdvisor may support multiple container runtimes (docker, rkt, cri-o, systemd, etc.)

As long as the kubelet continues to run cAdvisor, runtimes with native cAdvisor support may not want to run multiple monitoring agents to avoid performance regression in production.  Pending kubelet running a more light-weight monitoring solution, this PR allows remote runtimes to have their stats pulled from cAdvisor when cAdvisor is registered stats provider by introspection of the runtime endpoint.

See issue https://github.com/kubernetes/kubernetes/issues/51798

**Special notes for your reviewer**:
cAdvisor will be bumped to pick up https://github.com/google/cadvisor/pull/1741

At that time, CRI-O will support fetching stats from cAdvisor.

**Release note**:
```release-note
NONE
```

cmd/kubelet/app/server.go

@@ -393,7 +393,8 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) {
 	}
 
 	if kubeDeps.CAdvisorInterface == nil {
-		kubeDeps.CAdvisorInterface, err = cadvisor.New(s.Address, uint(s.CAdvisorPort), s.ContainerRuntime, s.RootDirectory)
+		imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntime, s.RemoteRuntimeEndpoint)
+		kubeDeps.CAdvisorInterface, err = cadvisor.New(s.Address, uint(s.CAdvisorPort), imageFsInfoProvider, s.RootDirectory)
 		if err != nil {
 			return err
 		}

pkg/kubelet/cadvisor/BUILD

@@ -26,6 +26,7 @@ go_library(
         "//pkg/api/v1/helper:go_default_library",
         "//pkg/features:go_default_library",
         "//vendor/github.com/google/cadvisor/events:go_default_library",
+        "//vendor/github.com/google/cadvisor/fs:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
@@ -37,7 +38,6 @@ go_library(
             "//vendor/github.com/golang/glog:go_default_library",
             "//vendor/github.com/google/cadvisor/cache/memory:go_default_library",
             "//vendor/github.com/google/cadvisor/container:go_default_library",
-            "//vendor/github.com/google/cadvisor/fs:go_default_library",
             "//vendor/github.com/google/cadvisor/http:go_default_library",
             "//vendor/github.com/google/cadvisor/manager:go_default_library",
             "//vendor/github.com/google/cadvisor/metrics:go_default_library",

pkg/kubelet/cadvisor/cadvisor_linux.go

@@ -32,7 +32,6 @@ import (
 	"github.com/google/cadvisor/cache/memory"
 	cadvisormetrics "github.com/google/cadvisor/container"
 	"github.com/google/cadvisor/events"
-	cadvisorfs "github.com/google/cadvisor/fs"
 	cadvisorhttp "github.com/google/cadvisor/http"
 	cadvisorapi "github.com/google/cadvisor/info/v1"
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
@@ -44,7 +43,7 @@ import (
 )
 
 type cadvisorClient struct {
-	runtime  string
+	imageFsInfoProvider ImageFsInfoProvider
 	rootPath            string
 	manager.Manager
 }
@@ -106,7 +105,7 @@ func containerLabels(c *cadvisorapi.ContainerInfo) map[string]string {
 }
 
 // New creates a cAdvisor and exports its API on the specified port if port > 0.
-func New(address string, port uint, runtime string, rootPath string) (Interface, error) {
+func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string) (Interface, error) {
 	sysFs := sysfs.NewRealSysFs()
 
 	// Create and start the cAdvisor container manager.
@@ -126,7 +125,7 @@ func New(address string, port uint, runtime string, rootPath string) (Interface,
 	}
 
 	cadvisorClient := &cadvisorClient{
-		runtime:  runtime,
+		imageFsInfoProvider: imageFsInfoProvider,
 		rootPath:            rootPath,
 		Manager:             m,
 	}
@@ -208,17 +207,10 @@ func (cc *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) {
 }
 
 func (cc *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
-	var label string
+	label, err := cc.imageFsInfoProvider.ImageFsInfoLabel()
-
+	if err != nil {
-	switch cc.runtime {
+		return cadvisorapiv2.FsInfo{}, err
-	case "docker":
-		label = cadvisorfs.LabelDockerImages
-	case "rkt":
-		label = cadvisorfs.LabelRktImages
-	default:
-		return cadvisorapiv2.FsInfo{}, fmt.Errorf("ImagesFsInfo: unknown runtime: %v", cc.runtime)
 	}
-
 	return cc.getFsInfo(label)
 }
 

pkg/kubelet/cadvisor/cadvisor_unsupported.go

@@ -31,7 +31,7 @@ type cadvisorUnsupported struct {
 
 var _ Interface = new(cadvisorUnsupported)
 
-func New(address string, port uint, runtime string, rootPath string) (Interface, error) {
+func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string) (Interface, error) {
 	return &cadvisorUnsupported{}, nil
 }
 

pkg/kubelet/cadvisor/cadvisor_windows.go

@@ -30,7 +30,7 @@ type cadvisorClient struct {
 var _ Interface = new(cadvisorClient)
 
 // New creates a cAdvisor and exports its API on the specified port if port > 0.
-func New(address string, port uint, runtime string, rootPath string) (Interface, error) {
+func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string) (Interface, error) {
 	return &cadvisorClient{}, nil
 }
 

pkg/kubelet/cadvisor/types.go

@@ -33,7 +33,7 @@ type Interface interface {
 
 	VersionInfo() (*cadvisorapi.VersionInfo, error)
 
-	// Returns usage information about the filesystem holding Docker images.
+	// Returns usage information about the filesystem holding container images.
 	ImagesFsInfo() (cadvisorapiv2.FsInfo, error)
 
 	// Returns usage information about the root filesystem.
@@ -49,3 +49,9 @@ type Interface interface {
 	// uuid.
 	GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error)
 }
+
+// ImageFsInfoProvider informs cAdvisor how to find imagefs for container images.
+type ImageFsInfoProvider interface {
+	// ImageFsInfoLabel returns the label cAdvisor should use to find the filesystem holding container images.
+	ImageFsInfoLabel() (string, error)
+}

pkg/kubelet/cadvisor/util.go

@@ -17,6 +17,9 @@ limitations under the License.
 package cadvisor
 
 import (
+	"fmt"
+
+	cadvisorfs "github.com/google/cadvisor/fs"
 	cadvisorapi "github.com/google/cadvisor/info/v1"
 	cadvisorapi2 "github.com/google/cadvisor/info/v2"
 	"k8s.io/api/core/v1"
@@ -26,6 +29,35 @@ import (
 	"k8s.io/kubernetes/pkg/features"
 )
 
+// imageFsInfoProvider knows how to translate the configured runtime
+// to its file system label for images.
+type imageFsInfoProvider struct {
+	runtime         string
+	runtimeEndpoint string
+}
+
+// ImageFsInfoLabel returns the image fs label for the configured runtime.
+// For remote runtimes, it handles additional runtimes natively understood by cAdvisor.
+func (i *imageFsInfoProvider) ImageFsInfoLabel() (string, error) {
+	switch i.runtime {
+	case "docker":
+		return cadvisorfs.LabelDockerImages, nil
+	case "rkt":
+		return cadvisorfs.LabelRktImages, nil
+	case "remote":
+		// TODO: pending rebase including https://github.com/google/cadvisor/pull/1741
+		if i.runtimeEndpoint == "/var/run/crio.sock" {
+			return "crio-images", nil
+		}
+	}
+	return "", fmt.Errorf("no imagefs label for configured runtime")
+}
+
+// NewImageFsInfoProvider returns a provider for the specified runtime configuration.
+func NewImageFsInfoProvider(runtime, runtimeEndpoint string) ImageFsInfoProvider {
+	return &imageFsInfoProvider{runtime: runtime, runtimeEndpoint: runtimeEndpoint}
+}
+
 func CapacityFromMachineInfo(info *cadvisorapi.MachineInfo) v1.ResourceList {
 	c := v1.ResourceList{
 		v1.ResourceCPU: *resource.NewMilliQuantity(

test/e2e_node/environment/conformance.go

@@ -99,7 +99,7 @@ func containerRuntime() error {
 	}
 
 	// Setup cadvisor to check the container environment
-	c, err := cadvisor.New("", 0 /*don't start the http server*/, "docker", "/var/lib/kubelet")
+	c, err := cadvisor.New("", 0 /*don't start the http server*/, cadvisor.NewImageFsInfoProvider("docker", ""), "/var/lib/kubelet")
 	if err != nil {
 		return printError("Container Runtime Check: %s Could not start cadvisor %v", failed, err)
 	}