diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index f93395b03ac..a29180214ee 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -583,93 +583,93 @@ }, { "ImportPath": "github.com/google/cadvisor/api", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/cache/memory", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/collector", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/container", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/events", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/fs", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/healthz", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/http", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/info/v1", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/info/v2", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/manager", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/metrics", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/pages", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/storage", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/summary", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/utils", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/validate", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/cadvisor/version", - "Comment": "v0.20.5", - "Rev": "9aa348ff5e191fcf3eccd59e5a434022aca77b87" + "Comment": "v0.21.1", + "Rev": "49a08d5139ae0111757a110ea3b55fe9a607873d" }, { "ImportPath": "github.com/google/gofuzz", diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go b/Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go index b567861e137..a4b39cdadab 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/api/versions.go @@ -467,7 +467,7 @@ func (self *version2_1) Version() string { } func (self *version2_1) SupportedRequestTypes() []string { - return self.baseVersion.SupportedRequestTypes() + return append([]string{machineStatsApi}, self.baseVersion.SupportedRequestTypes()...) } func (self *version2_1) HandleRequest(requestType string, request []string, m manager.Manager, w http.ResponseWriter, r *http.Request) error { @@ -492,9 +492,16 @@ func (self *version2_1) HandleRequest(requestType string, request []string, m ma if err != nil { return err } - contStats := make(map[string][]*v2.ContainerStats, len(conts)) + contStats := make(map[string]v2.ContainerInfo, len(conts)) for name, cont := range conts { - contStats[name] = v2.ContainerStatsFromV1(&cont.Spec, cont.Stats) + if name == "/" { + // Root cgroup stats should be exposed as machine stats + continue + } + contStats[name] = v2.ContainerInfo{ + Spec: v2.ContainerSpecFromV1(&cont.Spec, cont.Aliases, cont.Namespace), + Stats: v2.ContainerStatsFromV1(&cont.Spec, cont.Stats), + } } return writeResult(contStats, w) default: diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go index de73365dd1f..1518d0e3eac 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/factory.go @@ -40,6 +40,7 @@ var ArgDockerEndpoint = flag.String("docker", "unix:///var/run/docker.sock", "do var DockerNamespace = "docker" // Basepath to all container specific information that libcontainer stores. +// TODO: Deprecate this flag var dockerRootDir = flag.String("docker_root", "/var/lib/docker", "Absolute path to the Docker state root directory (default: /var/lib/docker)") var dockerRunDir = flag.String("docker_run", "/var/run/docker", "Absolute path to the Docker run directory (default: /var/run/docker)") @@ -61,6 +62,10 @@ func DockerStateDir() string { var useSystemd = false var check = sync.Once{} +const ( + dockerRootDirKey = "Root Dir" +) + func UseSystemd() bool { check.Do(func() { if *noSystemd { @@ -101,6 +106,7 @@ type dockerFactory struct { machineInfoFactory info.MachineInfoFactory storageDriver storageDriver + storageDir string client *docker.Client @@ -109,6 +115,8 @@ type dockerFactory struct { // Information about mounted filesystems. fsInfo fs.FsInfo + + dockerVersion []int } func (self *dockerFactory) String() string { @@ -129,9 +137,11 @@ func (self *dockerFactory) NewContainerHandler(name string, inHostNamespace bool self.machineInfoFactory, self.fsInfo, self.storageDriver, + self.storageDir, &self.cgroupSubsystems, inHostNamespace, metadataEnvs, + self.dockerVersion, ) return } @@ -214,36 +224,45 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo) error { if err != nil { return fmt.Errorf("unable to communicate with docker daemon: %v", err) } + var dockerVersion []int if version, err := client.Version(); err != nil { return fmt.Errorf("unable to communicate with docker daemon: %v", err) } else { expected_version := []int{1, 0, 0} version_string := version.Get("Version") - version, err := parseDockerVersion(version_string) + dockerVersion, err = parseDockerVersion(version_string) if err != nil { return fmt.Errorf("couldn't parse docker version: %v", err) } - for index, number := range version { + for index, number := range dockerVersion { if number > expected_version[index] { break } else if number < expected_version[index] { - return fmt.Errorf("cAdvisor requires docker version %v or above but we have found version %v reported as \"%v\"", expected_version, version, version_string) + return fmt.Errorf("cAdvisor requires docker version %v or above but we have found version %v reported as \"%v\"", expected_version, dockerVersion, version_string) } } } - // Check that the libcontainer execdriver is used. - information, err := DockerInfo() + information, err := client.Info() if err != nil { return fmt.Errorf("failed to detect Docker info: %v", err) } - execDriver, ok := information["ExecutionDriver"] - if !ok || !strings.HasPrefix(execDriver, "native") { + + // Check that the libcontainer execdriver is used. + execDriver := information.Get("ExecutionDriver") + if !strings.HasPrefix(execDriver, "native") { return fmt.Errorf("docker found, but not using native exec driver") } - sd, _ := information["Driver"] + sd := information.Get("Driver") + if sd == "" { + return fmt.Errorf("failed to find docker storage driver") + } + storageDir := information.Get("DockerRootDir") + if storageDir == "" { + storageDir = *dockerRootDir + } cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() if err != nil { return fmt.Errorf("failed to get cgroup subsystems: %v", err) @@ -251,11 +270,13 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo) error { glog.Infof("Registering Docker factory") f := &dockerFactory{ - machineInfoFactory: factory, - client: client, - storageDriver: storageDriver(sd), cgroupSubsystems: cgroupSubsystems, + client: client, + dockerVersion: dockerVersion, fsInfo: fsInfo, + machineInfoFactory: factory, + storageDriver: storageDriver(sd), + storageDir: storageDir, } container.RegisterContainerHandlerFactory(f) return nil diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go index 9222d8df46a..36c4907897a 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/container/docker/handler.go @@ -17,6 +17,7 @@ package docker import ( "fmt" + "io/ioutil" "math" "path" "strings" @@ -35,14 +36,10 @@ import ( ) const ( - // Path to aufs dir where all the files exist. - // aufs/layers is ignored here since it does not hold a lot of data. - // aufs/mnt contains the mount points used to compose the rootfs. Hence it is also ignored. - pathToAufsDir = "aufs/diff" + // The read write layers exist here. + aufsRWLayer = "diff" // Path to the directory where docker stores log files if the json logging driver is enabled. pathToContainersDir = "containers" - // Path to the overlayfs storage driver directory. - pathToOverlayDir = "overlay" ) type dockerContainerHandler struct { @@ -86,15 +83,34 @@ type dockerContainerHandler struct { fsHandler fsHandler } +func getRwLayerID(containerID, storageDir string, sd storageDriver, dockerVersion []int) (string, error) { + const ( + // Docker version >=1.10.0 have a randomized ID for the root fs of a container. + randomizedRWLayerMinorVersion = 10 + rwLayerIDFile = "mount-id" + ) + if (dockerVersion[0] <= 1) && (dockerVersion[1] < randomizedRWLayerMinorVersion) { + return containerID, nil + } + + bytes, err := ioutil.ReadFile(path.Join(storageDir, "image", string(sd), "layerdb", "mounts", containerID, rwLayerIDFile)) + if err != nil { + return "", fmt.Errorf("failed to identify the read-write layer ID for container %q. - %v", containerID, err) + } + return string(bytes), err +} + func newDockerContainerHandler( client *docker.Client, name string, machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, storageDriver storageDriver, + storageDir string, cgroupSubsystems *containerlibcontainer.CgroupSubsystems, inHostNamespace bool, metadataEnvs []string, + dockerVersion []int, ) (container.ContainerHandler, error) { // Create the cgroup paths. cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) @@ -118,13 +134,18 @@ func newDockerContainerHandler( id := ContainerNameToDockerId(name) // Add the Containers dir where the log files are stored. - otherStorageDir := path.Join(*dockerRootDir, pathToContainersDir, id) + otherStorageDir := path.Join(storageDir, pathToContainersDir, id) + + rwLayerID, err := getRwLayerID(id, storageDir, storageDriver, dockerVersion) + if err != nil { + return nil, err + } var rootfsStorageDir string switch storageDriver { case aufsStorageDriver: - rootfsStorageDir = path.Join(*dockerRootDir, pathToAufsDir, id) + rootfsStorageDir = path.Join(storageDir, string(aufsStorageDriver), aufsRWLayer, rwLayerID) case overlayStorageDriver: - rootfsStorageDir = path.Join(*dockerRootDir, pathToOverlayDir, id) + rootfsStorageDir = path.Join(storageDir, string(overlayStorageDriver), rwLayerID) } handler := &dockerContainerHandler{ @@ -180,9 +201,11 @@ func (self *dockerContainerHandler) Cleanup() { func (self *dockerContainerHandler) ContainerReference() (info.ContainerReference, error) { return info.ContainerReference{ + Id: self.id, Name: self.name, Aliases: self.aliases, Namespace: DockerNamespace, + Labels: self.labels, }, nil } diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go b/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go index 6b7f40de1fc..bb4fc322e23 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/info/v1/container.go @@ -70,6 +70,9 @@ type ContainerSpec struct { // Container reference contains enough information to uniquely identify a container type ContainerReference struct { + // The container id + Id string `json:"id,omitempty"` + // The absolute name of the container. This is unique on the machine. Name string `json:"name"` @@ -80,6 +83,8 @@ type ContainerReference struct { // Namespace under which the aliases of a container are unique. // An example of a namespace is "docker" for Docker containers. Namespace string `json:"namespace,omitempty"` + + Labels map[string]string `json:"labels,omitempty"` } // Sorts by container name. diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/conversion.go b/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/conversion.go index fe241733158..56aef777caf 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/conversion.go +++ b/Godeps/_workspace/src/github.com/google/cadvisor/info/v2/conversion.go @@ -123,7 +123,7 @@ func ContainerStatsFromV1(spec *v1.ContainerSpec, stats []*v1.ContainerStats) [] } } else if len(val.Filesystem) > 1 { // Cannot handle multiple devices per container. - glog.Errorf("failed to handle multiple devices for container. Skipping Filesystem stats") + glog.V(2).Infof("failed to handle multiple devices for container. Skipping Filesystem stats") } } if spec.HasDiskIo { diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/storage/kafka/kafka.go b/Godeps/_workspace/src/github.com/google/cadvisor/storage/kafka/kafka.go new file mode 100644 index 00000000000..e43a374c81d --- /dev/null +++ b/Godeps/_workspace/src/github.com/google/cadvisor/storage/kafka/kafka.go @@ -0,0 +1,114 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "encoding/json" + "flag" + "os" + "strings" + "time" + + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/storage" + "github.com/google/cadvisor/utils/container" + + kafka "github.com/Shopify/sarama" + "github.com/golang/glog" +) + +func init() { + storage.RegisterStorageDriver("kafka", new) +} + +var ( + brokers = flag.String("storage_driver_kafka_broker_list", "localhost:9092", "kafka broker(s) csv") + topic = flag.String("storage_driver_kafka_topic", "stats", "kafka topic") +) + +type kafkaStorage struct { + producer kafka.AsyncProducer + topic string + machineName string +} + +type detailSpec struct { + Timestamp time.Time `json:"timestamp"` + MachineName string `json:"machine_name,omitempty"` + ContainerName string `json:"container_Name,omitempty"` + ContainerID string `json:"container_Id,omitempty"` + ContainerLabels map[string]string `json:"container_labels,omitempty"` + ContainerStats *info.ContainerStats `json:"container_stats,omitempty"` +} + +func (driver *kafkaStorage) infoToDetailSpec(ref info.ContainerReference, stats *info.ContainerStats) *detailSpec { + timestamp := time.Now() + containerID := ref.Id + containerLabels := ref.Labels + containerName := container.GetPreferredName(ref) + + detail := &detailSpec{ + Timestamp: timestamp, + MachineName: driver.machineName, + ContainerName: containerName, + ContainerID: containerID, + ContainerLabels: containerLabels, + ContainerStats: stats, + } + return detail +} + +func (driver *kafkaStorage) AddStats(ref info.ContainerReference, stats *info.ContainerStats) error { + detail := driver.infoToDetailSpec(ref, stats) + b, err := json.Marshal(detail) + + driver.producer.Input() <- &kafka.ProducerMessage{ + Topic: driver.topic, + Value: kafka.StringEncoder(b), + } + + return err +} + +func (self *kafkaStorage) Close() error { + return self.producer.Close() +} + +func new() (storage.StorageDriver, error) { + machineName, err := os.Hostname() + if err != nil { + return nil, err + } + return newStorage(machineName) +} + +func newStorage(machineName string) (storage.StorageDriver, error) { + config := kafka.NewConfig() + config.Producer.RequiredAcks = kafka.WaitForAll + + brokerList := strings.Split(*brokers, ",") + glog.V(4).Infof("Kafka brokers:%q", brokers) + + producer, err := kafka.NewAsyncProducer(brokerList, config) + if err != nil { + return nil, err + } + ret := &kafkaStorage{ + producer: producer, + topic: *topic, + machineName: machineName, + } + return ret, nil +} diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/utils/container/container.go b/Godeps/_workspace/src/github.com/google/cadvisor/utils/container/container.go new file mode 100644 index 00000000000..ea956a5c6e9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/google/cadvisor/utils/container/container.go @@ -0,0 +1,31 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package container + +import ( + info "github.com/google/cadvisor/info/v1" +) + +// Returns the alias a container is known by within a certain namespace, +// if available. Otherwise returns the absolute name of the container. +func GetPreferredName(ref info.ContainerReference) string { + var containerName string + if len(ref.Aliases) > 0 { + containerName = ref.Aliases[0] + } else { + containerName = ref.Name + } + return containerName +} diff --git a/Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION b/Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION index 58d8f8d4a98..b9f8e558df4 100644 --- a/Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION +++ b/Godeps/_workspace/src/github.com/google/cadvisor/version/VERSION @@ -1 +1 @@ -0.20.5 \ No newline at end of file +0.21.1 \ No newline at end of file