Merge pull request #53107 from Random-Liu/fix-cri-stats

Automatic merge from submit-queue (batch tested with PRs 53234, 53252, 53267, 53276, 53107). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Fix imagefs stats

Without this CRI stats based summary api won't work:
```console
$ curl localhost:10255/stats/summary
Internal Error: failed to get root cgroup stats: failed to get imageFs info: no imagefs label for configured runtime
```
With this PR, we could get summary api from cri-containerd now:
```console
$ curl localhost:10255/stats/summary
{
  "node": {
   "nodeName": "127.0.0.1",
   "startTime": "2017-09-23T06:26:49Z",
   "cpu": {
    "time": "2017-09-27T05:12:08Z",
    "usageNanoCores": 275510572,
    "usageCoreNanoSeconds": 11924595625329
   },
   "memory": {
    "time": "2017-09-27T05:12:08Z",
    "availableBytes": 27737075712,
    "usageBytes": 6028234752,
    "workingSetBytes": 3884470272,
    "rssBytes": 652304384,
    "pageFaults": 98472,
    "majorPageFaults": 87
   },
   "fs": {
    "time": "2017-09-27T05:12:08Z",
    "availableBytes": 75281231872,
    "capacityBytes": 104022159360,
    "usedBytes": 28724150272,
    "inodesFree": 12003204,
    "inodes": 12800000,
    "inodesUsed": 796796
   },
   "runtime": {
    "imageFs": {
     "time": "2017-09-27T05:12:00Z",
     "availableBytes": 75281231872,
     "capacityBytes": 104022159360,
     "usedBytes": 247732356,
     "inodesFree": 12003204,
     "inodes": 12800000,
     "inodesUsed": 6103
    }
   }
  },
  "pods": [
   {
    "podRef": {
     "name": "kube-dns-7797cb8758-qxkrz",
     "namespace": "kube-system",
     "uid": "4425b069-a342-11e7-ac90-42010af00002"
    },
    "startTime": "2017-09-27T05:11:23Z",
    "containers": [
     {
      "name": "kubedns",
      "startTime": "2017-09-27T05:11:24Z",
      "cpu": {
       "time": "1970-01-01T00:00:01Z",
       "usageCoreNanoSeconds": 154194917
      },
      "memory": {
       "time": "1970-01-01T00:00:01Z",
       "workingSetBytes": 7643136
      },
      "rootfs": {
       "time": "2017-09-27T05:12:00Z",
       "availableBytes": 75281231872,
       "capacityBytes": 104022159360,
       "usedBytes": 9,
       "inodesFree": 12003204,
       "inodes": 12800000,
       "inodesUsed": 32768
      },
      "logs": {
       "time": "2017-09-27T05:12:08Z",
       "availableBytes": 75281231872,
       "capacityBytes": 104022159360,
       "inodesFree": 12003204,
       "inodes": 12800000
      },
      "userDefinedMetrics": null
     },
     {
      "name": "dnsmasq",
      "startTime": "2017-09-27T05:11:24Z",
      "cpu": {
       "time": "1970-01-01T00:00:01Z",
       "usageCoreNanoSeconds": 114482989
      },
      "memory": {
       "time": "1970-01-01T00:00:01Z",
       "workingSetBytes": 7966720
      },
      "rootfs": {
       "time": "2017-09-27T05:12:00Z",
       "availableBytes": 75281231872,
       "capacityBytes": 104022159360,
       "usedBytes": 9,
       "inodesFree": 12003204,
       "inodes": 12800000,
       "inodesUsed": 28675
      },
      "logs": {
       "time": "2017-09-27T05:12:08Z",
       "availableBytes": 75281231872,
       "capacityBytes": 104022159360,
       "inodesFree": 12003204,
       "inodes": 12800000
      },
      "userDefinedMetrics": null
     },
     {
      "name": "sidecar",
      "startTime": "2017-09-27T05:11:24Z",
      "cpu": {
       "time": "1970-01-01T00:00:01Z",
       "usageCoreNanoSeconds": 140797580
      },
      "memory": {
       "time": "1970-01-01T00:00:01Z",
       "workingSetBytes": 7430144
      },
      "rootfs": {
       "time": "2017-09-27T05:12:00Z",
       "availableBytes": 75281231872,
       "capacityBytes": 104022159360,
       "usedBytes": 8,
       "inodesFree": 12003204,
       "inodes": 12800000,
       "inodesUsed": 28672
      },
      "logs": {
       "time": "2017-09-27T05:12:08Z",
       "availableBytes": 75281231872,
       "capacityBytes": 104022159360,
       "inodesFree": 12003204,
       "inodes": 12800000
      },
      "userDefinedMetrics": null
     }
    ],
    "volume": [
     {
      "time": "2017-09-27T05:12:03Z",
      "availableBytes": 15810760704,
      "capacityBytes": 15810772992,
      "usedBytes": 12288,
      "inodesFree": 3860043,
      "inodes": 3860052,
      "inodesUsed": 9,
      "name": "kube-dns-token-l2blr"
     }
    ]
   }
  ]
 }
```
Signed-off-by: Lantao Liu <lantaol@google.com>

```release-note
Fix the bug that query Kubelet's stats summary with CRI stats enabled results in error.
```

pkg/kubelet/stats/helper.go

@@ -74,46 +74,53 @@ func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo
 		}
 	}
 
-	// The container logs live on the node rootfs device
+	if rootFs != nil {
-	result.Logs = &statsapi.FsStats{
+		// The container logs live on the node rootfs device
-		Time:           metav1.NewTime(cstat.Timestamp),
+		result.Logs = &statsapi.FsStats{
-		AvailableBytes: &rootFs.Available,
+			Time:           metav1.NewTime(cstat.Timestamp),
-		CapacityBytes:  &rootFs.Capacity,
+			AvailableBytes: &rootFs.Available,
-		InodesFree:     rootFs.InodesFree,
+			CapacityBytes:  &rootFs.Capacity,
-		Inodes:         rootFs.Inodes,
+			InodesFree:     rootFs.InodesFree,
+			Inodes:         rootFs.Inodes,
+		}
+
+		if rootFs.Inodes != nil && rootFs.InodesFree != nil {
+			logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree
+			result.Logs.InodesUsed = &logsInodesUsed
+		}
 	}
 
-	if rootFs.Inodes != nil && rootFs.InodesFree != nil {
+	if imageFs != nil {
-		logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree
+		// The container rootFs lives on the imageFs devices (which may not be the node root fs)
-		result.Logs.InodesUsed = &logsInodesUsed
+		result.Rootfs = &statsapi.FsStats{
-	}
+			Time:           metav1.NewTime(cstat.Timestamp),
-
+			AvailableBytes: &imageFs.Available,
-	// The container rootFs lives on the imageFs devices (which may not be the node root fs)
+			CapacityBytes:  &imageFs.Capacity,
-	result.Rootfs = &statsapi.FsStats{
+			InodesFree:     imageFs.InodesFree,
-		Time:           metav1.NewTime(cstat.Timestamp),
+			Inodes:         imageFs.Inodes,
-		AvailableBytes: &imageFs.Available,
+		}
-		CapacityBytes:  &imageFs.Capacity,
-		InodesFree:     imageFs.InodesFree,
-		Inodes:         imageFs.Inodes,
 	}
 
 	cfs := cstat.Filesystem
 	if cfs != nil {
 		if cfs.BaseUsageBytes != nil {
-			rootfsUsage := *cfs.BaseUsageBytes
+			if result.Rootfs != nil {
-			result.Rootfs.UsedBytes = &rootfsUsage
+				rootfsUsage := *cfs.BaseUsageBytes
-			if cfs.TotalUsageBytes != nil {
+				result.Rootfs.UsedBytes = &rootfsUsage
+			}
+			if cfs.TotalUsageBytes != nil && result.Logs != nil {
 				logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes
 				result.Logs.UsedBytes = &logsUsage
 			}
 		}
-		if cfs.InodeUsage != nil {
+		if cfs.InodeUsage != nil && result.Rootfs != nil {
 			rootInodes := *cfs.InodeUsage
 			result.Rootfs.InodesUsed = &rootInodes
 		}
 	}
 
 	result.UserDefinedMetrics = cadvisorInfoToUserDefinedMetrics(info)
+
 	return result
 }
 

pkg/kubelet/stats/stats_provider.go

@@ -86,21 +86,15 @@ type containerStatsProvider interface {
 	ImageFsStats() (*statsapi.FsStats, error)
 }
 
-// GetCgroupStats returns the stats of the cgroup with the cgroupName.
+// GetCgroupStats returns the stats of the cgroup with the cgroupName. Note that
+// this function doesn't generate filesystem stats.
 func (p *StatsProvider) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
 	info, err := getCgroupInfo(p.cadvisor, cgroupName)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err)
 	}
-	rootFsInfo, err := p.cadvisor.RootFsInfo()
+	// Rootfs and imagefs doesn't make sense for raw cgroup.
-	if err != nil {
+	s := cadvisorInfoToContainerStats(cgroupName, info, nil, nil)
-		return nil, nil, fmt.Errorf("failed to get rootFs info: %v", err)
-	}
-	imageFsInfo, err := p.cadvisor.ImagesFsInfo()
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to get imageFs info: %v", err)
-	}
-	s := cadvisorInfoToContainerStats(cgroupName, info, &rootFsInfo, &imageFsInfo)
 	n := cadvisorInfoToNetworkStats(cgroupName, info)
 	return s, n, nil
 }

pkg/kubelet/stats/stats_provider_test.go

@@ -69,9 +69,7 @@ var (
 func TestGetCgroupStats(t *testing.T) {
 	const (
 		cgroupName        = "test-cgroup-name"
-		rootFsInfoSeed    = 1000
+		containerInfoSeed = 1000
-		imageFsInfoSeed   = 2000
-		containerInfoSeed = 3000
 	)
 	var (
 		mockCadvisor     = new(cadvisortest.Mock)
@@ -81,16 +79,11 @@ func TestGetCgroupStats(t *testing.T) {
 		assert  = assert.New(t)
 		options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false}
 
-		rootFsInfo       = getTestFsInfo(rootFsInfoSeed)
-		imageFsInfo      = getTestFsInfo(imageFsInfoSeed)
 		containerInfo    = getTestContainerInfo(containerInfoSeed, "test-pod", "test-ns", "test-container")
 		containerInfoMap = map[string]cadvisorapiv2.ContainerInfo{cgroupName: containerInfo}
 	)
 
-	mockCadvisor.
+	mockCadvisor.On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil)
-		On("RootFsInfo").Return(rootFsInfo, nil).
-		On("ImagesFsInfo").Return(imageFsInfo, nil).
-		On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil)
 
 	provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{})
 	cs, ns, err := provider.GetCgroupStats(cgroupName)
@@ -98,21 +91,11 @@ func TestGetCgroupStats(t *testing.T) {
 
 	checkCPUStats(t, "", containerInfoSeed, cs.CPU)
 	checkMemoryStats(t, "", containerInfoSeed, containerInfo, cs.Memory)
-	checkFsStats(t, "", imageFsInfoSeed, cs.Rootfs)
-	checkFsStats(t, "", rootFsInfoSeed, cs.Logs)
 	checkNetworkStats(t, "", containerInfoSeed, ns)
 
 	assert.Equal(cgroupName, cs.Name)
 	assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime)
 
-	assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Rootfs.Time)
-	assert.Equal(*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Rootfs.UsedBytes)
-	assert.Equal(*containerInfo.Stats[0].Filesystem.InodeUsage, *cs.Rootfs.InodesUsed)
-
-	assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Logs.Time)
-	assert.Equal(*containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Logs.UsedBytes)
-	assert.Equal(*rootFsInfo.Inodes-*rootFsInfo.InodesFree, *cs.Logs.InodesUsed)
-
 	mockCadvisor.AssertExpectations(t)
 }