Merge pull request #20621 from vishh/cadvisor-godeps

Auto commit by PR queue bot

Godeps/Godeps.json

@@ -583,93 +583,93 @@
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/api",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/cache/memory",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/collector",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/container",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/events",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/fs",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/healthz",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/http",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/info/v1",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/info/v2",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/manager",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/metrics",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/pages",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/storage",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/summary",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/utils",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/validate",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/cadvisor/version",
-			"Comment": "v0.20.5",
+			"Comment": "v0.21.1",
-			"Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87"
+			"Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d"
 		},
 		{
 			"ImportPath": "github.com/google/gofuzz",

Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go

@@ -467,7 +467,7 @@ func (self *version2_1) Version() string {
 }
 
 func (self *version2_1) SupportedRequestTypes() []string {
-	return self.baseVersion.SupportedRequestTypes()
+	return append([]string{machineStatsApi}, self.baseVersion.SupportedRequestTypes()...)
 }
 
 func (self *version2_1) HandleRequest(requestType string, request []string, m manager.Manager, w http.ResponseWriter, r *http.Request) error {
@@ -492,9 +492,16 @@ func (self *version2_1) HandleRequest(requestType string, request []string, m ma
 		if err != nil {
 			return err
 		}
-		contStats := make(map[string][]*v2.ContainerStats, len(conts))
+		contStats := make(map[string]v2.ContainerInfo, len(conts))
 		for name, cont := range conts {
-			contStats[name] = v2.ContainerStatsFromV1(&cont.Spec, cont.Stats)
+			if name == "/" {
+				// Root cgroup stats should be exposed as machine stats
+				continue
+			}
+			contStats[name] = v2.ContainerInfo{
+				Spec:  v2.ContainerSpecFromV1(&cont.Spec, cont.Aliases, cont.Namespace),
+				Stats: v2.ContainerStatsFromV1(&cont.Spec, cont.Stats),
+			}
 		}
 		return writeResult(contStats, w)
 	default:

Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go

@@ -40,6 +40,7 @@ var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "do
 var DockerNamespace = "docker"
 
 // Basepath to all container specific information that libcontainer stores.
+// TODO: Deprecate this flag
 var dockerRootDir = flag.String("docker_root", "/var/lib/docker", "Absolute path to the Docker state root directory (default: /var/lib/docker)")
 var dockerRunDir = flag.String("docker_run", "/var/run/docker", "Absolute path to the Docker run directory (default: /var/run/docker)")
 
@@ -61,6 +62,10 @@ func DockerStateDir() string {
 var useSystemd = false
 var check = sync.Once{}
 
+const (
+	dockerRootDirKey = "Root Dir"
+)
+
 func UseSystemd() bool {
 	check.Do(func() {
 		if *noSystemd {
@@ -101,6 +106,7 @@ type dockerFactory struct {
 	machineInfoFactory info.MachineInfoFactory
 
 	storageDriver storageDriver
+	storageDir    string
 
 	client *docker.Client
 
@@ -109,6 +115,8 @@ type dockerFactory struct {
 
 	// Information about mounted filesystems.
 	fsInfo fs.FsInfo
+
+	dockerVersion []int
 }
 
 func (self *dockerFactory) String() string {
@@ -129,9 +137,11 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool
 		self.machineInfoFactory,
 		self.fsInfo,
 		self.storageDriver,
+		self.storageDir,
 		&self.cgroupSubsystems,
 		inHostNamespace,
 		metadataEnvs,
+		self.dockerVersion,
 	)
 	return
 }
@@ -214,36 +224,45 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo) error {
 	if err != nil {
 		return fmt.Errorf("unable to communicate with docker daemon: %v", err)
 	}
+	var dockerVersion []int
 	if version, err := client.Version(); err != nil {
 		return fmt.Errorf("unable to communicate with docker daemon: %v", err)
 	} else {
 		expected_version := []int{1, 0, 0}
 		version_string := version.Get("Version")
-		version, err := parseDockerVersion(version_string)
+		dockerVersion, err = parseDockerVersion(version_string)
 		if err != nil {
 			return fmt.Errorf("couldn't parse docker version: %v", err)
 		}
-		for index, number := range version {
+		for index, number := range dockerVersion {
 			if number > expected_version[index] {
 				break
 			} else if number < expected_version[index] {
-				return fmt.Errorf("cAdvisor requires docker version %v or above but we have found version %v reported as \"%v\"", expected_version, version, version_string)
+				return fmt.Errorf("cAdvisor requires docker version %v or above but we have found version %v reported as \"%v\"", expected_version, dockerVersion, version_string)
 			}
 		}
 	}
 
-	// Check that the libcontainer execdriver is used.
+	information, err := client.Info()
-	information, err := DockerInfo()
 	if err != nil {
 		return fmt.Errorf("failed to detect Docker info: %v", err)
 	}
-	execDriver, ok := information["ExecutionDriver"]
+
-	if !ok || !strings.HasPrefix(execDriver, "native") {
+	// Check that the libcontainer execdriver is used.
+	execDriver := information.Get("ExecutionDriver")
+	if !strings.HasPrefix(execDriver, "native") {
 		return fmt.Errorf("docker found, but not using native exec driver")
 	}
 
-	sd, _ := information["Driver"]
+	sd := information.Get("Driver")
+	if sd == "" {
+		return fmt.Errorf("failed to find docker storage driver")
+	}
 
+	storageDir := information.Get("DockerRootDir")
+	if storageDir == "" {
+		storageDir = *dockerRootDir
+	}
 	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
 	if err != nil {
 		return fmt.Errorf("failed to get cgroup subsystems: %v", err)
@@ -251,11 +270,13 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo) error {
 
 	glog.Infof("Registering Docker factory")
 	f := &dockerFactory{
-		machineInfoFactory: factory,
-		client:             client,
-		storageDriver:      storageDriver(sd),
 		cgroupSubsystems:   cgroupSubsystems,
+		client:             client,
+		dockerVersion:      dockerVersion,
 		fsInfo:             fsInfo,
+		machineInfoFactory: factory,
+		storageDriver:      storageDriver(sd),
+		storageDir:         storageDir,
 	}
 	container.RegisterContainerHandlerFactory(f)
 	return nil

Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go

@@ -17,6 +17,7 @@ package docker
 
 import (
 	"fmt"
+	"io/ioutil"
 	"math"
 	"path"
 	"strings"
@@ -35,14 +36,10 @@ import (
 )
 
 const (
-	// Path to aufs dir where all the files exist.
+	// The read write layers exist here.
-	// aufs/layers is ignored here since it does not hold a lot of data.
+	aufsRWLayer = "diff"
-	// aufs/mnt contains the mount points used to compose the rootfs. Hence it is also ignored.
-	pathToAufsDir = "aufs/diff"
 	// Path to the directory where docker stores log files if the json logging driver is enabled.
 	pathToContainersDir = "containers"
-	// Path to the overlayfs storage driver directory.
-	pathToOverlayDir = "overlay"
 )
 
 type dockerContainerHandler struct {
@@ -86,15 +83,34 @@ type dockerContainerHandler struct {
 	fsHandler fsHandler
 }
 
+func getRwLayerID(containerID, storageDir string, sd storageDriver, dockerVersion []int) (string, error) {
+	const (
+		// Docker version >=1.10.0 have a randomized ID for the root fs of a container.
+		randomizedRWLayerMinorVersion = 10
+		rwLayerIDFile                 = "mount-id"
+	)
+	if (dockerVersion[0] <= 1) && (dockerVersion[1] < randomizedRWLayerMinorVersion) {
+		return containerID, nil
+	}
+
+	bytes, err := ioutil.ReadFile(path.Join(storageDir, "image", string(sd), "layerdb", "mounts", containerID, rwLayerIDFile))
+	if err != nil {
+		return "", fmt.Errorf("failed to identify the read-write layer ID for container %q. - %v", containerID, err)
+	}
+	return string(bytes), err
+}
+
 func newDockerContainerHandler(
 	client *docker.Client,
 	name string,
 	machineInfoFactory info.MachineInfoFactory,
 	fsInfo fs.FsInfo,
 	storageDriver storageDriver,
+	storageDir string,
 	cgroupSubsystems *containerlibcontainer.CgroupSubsystems,
 	inHostNamespace bool,
 	metadataEnvs []string,
+	dockerVersion []int,
 ) (container.ContainerHandler, error) {
 	// Create the cgroup paths.
 	cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints))
@@ -118,13 +134,18 @@ func newDockerContainerHandler(
 	id := ContainerNameToDockerId(name)
 
 	// Add the Containers dir where the log files are stored.
-	otherStorageDir := path.Join(*dockerRootDir, pathToContainersDir, id)
+	otherStorageDir := path.Join(storageDir, pathToContainersDir, id)
+
+	rwLayerID, err := getRwLayerID(id, storageDir, storageDriver, dockerVersion)
+	if err != nil {
+		return nil, err
+	}
 	var rootfsStorageDir string
 	switch storageDriver {
 	case aufsStorageDriver:
-		rootfsStorageDir = path.Join(*dockerRootDir, pathToAufsDir, id)
+		rootfsStorageDir = path.Join(storageDir, string(aufsStorageDriver), aufsRWLayer, rwLayerID)
 	case overlayStorageDriver:
-		rootfsStorageDir = path.Join(*dockerRootDir, pathToOverlayDir, id)
+		rootfsStorageDir = path.Join(storageDir, string(overlayStorageDriver), rwLayerID)
 	}
 
 	handler := &dockerContainerHandler{
@@ -180,9 +201,11 @@ func (self *dockerContainerHandler) Cleanup() {
 
 func (self *dockerContainerHandler) ContainerReference() (info.ContainerReference, error) {
 	return info.ContainerReference{
+		Id:        self.id,
 		Name:      self.name,
 		Aliases:   self.aliases,
 		Namespace: DockerNamespace,
+		Labels:    self.labels,
 	}, nil
 }
 

Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go

@@ -70,6 +70,9 @@ type ContainerSpec struct {
 
 // Container reference contains enough information to uniquely identify a container
 type ContainerReference struct {
+	// The container id
+	Id string `json:"id,omitempty"`
+
 	// The absolute name of the container. This is unique on the machine.
 	Name string `json:"name"`
 
@@ -80,6 +83,8 @@ type ContainerReference struct {
 	// Namespace under which the aliases of a container are unique.
 	// An example of a namespace is "docker" for Docker containers.
 	Namespace string `json:"namespace,omitempty"`
+
+	Labels map[string]string `json:"labels,omitempty"`
 }
 
 // Sorts by container name.

Godeps/_workspace/src/github.com/google/cadvisor/info/v2/conversion.go

@@ -123,7 +123,7 @@ func ContainerStatsFromV1(spec *v1.ContainerSpec, stats []*v1.ContainerStats) []
 				}
 			} else if len(val.Filesystem) > 1 {
 				// Cannot handle multiple devices per container.
-				glog.Errorf("failed to handle multiple devices for container. Skipping Filesystem stats")
+				glog.V(2).Infof("failed to handle multiple devices for container. Skipping Filesystem stats")
 			}
 		}
 		if spec.HasDiskIo {

Godeps/_workspace/src/github.com/google/cadvisor/storage/kafka/kafka.go

@@ -0,0 +1,114 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kafka
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"strings"
+	"time"
+
+	info "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/storage"
+	"github.com/google/cadvisor/utils/container"
+
+	kafka "github.com/Shopify/sarama"
+	"github.com/golang/glog"
+)
+
+func init() {
+	storage.RegisterStorageDriver("kafka", new)
+}
+
+var (
+	brokers = flag.String("storage_driver_kafka_broker_list", "localhost:9092", "kafka broker(s) csv")
+	topic   = flag.String("storage_driver_kafka_topic", "stats", "kafka topic")
+)
+
+type kafkaStorage struct {
+	producer    kafka.AsyncProducer
+	topic       string
+	machineName string
+}
+
+type detailSpec struct {
+	Timestamp       time.Time            `json:"timestamp"`
+	MachineName     string               `json:"machine_name,omitempty"`
+	ContainerName   string               `json:"container_Name,omitempty"`
+	ContainerID     string               `json:"container_Id,omitempty"`
+	ContainerLabels map[string]string    `json:"container_labels,omitempty"`
+	ContainerStats  *info.ContainerStats `json:"container_stats,omitempty"`
+}
+
+func (driver *kafkaStorage) infoToDetailSpec(ref info.ContainerReference, stats *info.ContainerStats) *detailSpec {
+	timestamp := time.Now()
+	containerID := ref.Id
+	containerLabels := ref.Labels
+	containerName := container.GetPreferredName(ref)
+
+	detail := &detailSpec{
+		Timestamp:       timestamp,
+		MachineName:     driver.machineName,
+		ContainerName:   containerName,
+		ContainerID:     containerID,
+		ContainerLabels: containerLabels,
+		ContainerStats:  stats,
+	}
+	return detail
+}
+
+func (driver *kafkaStorage) AddStats(ref info.ContainerReference, stats *info.ContainerStats) error {
+	detail := driver.infoToDetailSpec(ref, stats)
+	b, err := json.Marshal(detail)
+
+	driver.producer.Input() <- &kafka.ProducerMessage{
+		Topic: driver.topic,
+		Value: kafka.StringEncoder(b),
+	}
+
+	return err
+}
+
+func (self *kafkaStorage) Close() error {
+	return self.producer.Close()
+}
+
+func new() (storage.StorageDriver, error) {
+	machineName, err := os.Hostname()
+	if err != nil {
+		return nil, err
+	}
+	return newStorage(machineName)
+}
+
+func newStorage(machineName string) (storage.StorageDriver, error) {
+	config := kafka.NewConfig()
+	config.Producer.RequiredAcks = kafka.WaitForAll
+
+	brokerList := strings.Split(*brokers, ",")
+	glog.V(4).Infof("Kafka brokers:%q", brokers)
+
+	producer, err := kafka.NewAsyncProducer(brokerList, config)
+	if err != nil {
+		return nil, err
+	}
+	ret := &kafkaStorage{
+		producer:    producer,
+		topic:       *topic,
+		machineName: machineName,
+	}
+	return ret, nil
+}

Godeps/_workspace/src/github.com/google/cadvisor/utils/container/container.go

@@ -0,0 +1,31 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package container
+
+import (
+	info "github.com/google/cadvisor/info/v1"
+)
+
+// Returns the alias a container is known by within a certain namespace,
+// if available. Otherwise returns the absolute name of the container.
+func GetPreferredName(ref info.ContainerReference) string {
+	var containerName string
+	if len(ref.Aliases) > 0 {
+		containerName = ref.Aliases[0]
+	} else {
+		containerName = ref.Name
+	}
+	return containerName
+}

Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION

@@ -1 +1 @@
-0.20.5
+0.21.1