From 7404b7019cbbe7cd6267f2c0027e4d879a825262 Mon Sep 17 00:00:00 2001 From: Victor Marmol Date: Mon, 27 Apr 2015 13:03:55 -0700 Subject: [PATCH] Move Docker-specific log handling to DockerManager. Eases the separation of Docker-specific code into the Docker-specific runtime. --- cmd/integration/integration.go | 5 +- cmd/kubelet/app/server.go | 7 +- cmd/kubernetes/kubernetes.go | 4 +- pkg/kubelet/container/os.go | 55 ++++++++++++++ pkg/kubelet/dockertools/docker_test.go | 4 +- pkg/kubelet/dockertools/manager.go | 63 ++++++++++++++-- pkg/kubelet/kubelet.go | 99 ++++---------------------- pkg/kubelet/kubelet_test.go | 4 +- pkg/kubelet/pod_workers_test.go | 2 +- pkg/kubelet/runonce_test.go | 6 +- 10 files changed, 143 insertions(+), 106 deletions(-) create mode 100644 pkg/kubelet/container/os.go diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go index faf7bfffe94..b5e2e8c9280 100644 --- a/cmd/integration/integration.go +++ b/cmd/integration/integration.go @@ -46,6 +46,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/fields" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor" + kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" "github.com/GoogleCloudPlatform/kubernetes/pkg/master" @@ -230,14 +231,14 @@ func startComponents(firstManifestURL, secondManifestURL, apiVersion string) (st testRootDir := makeTempDirOrDie("kubelet_integ_1.", "") configFilePath := makeTempDirOrDie("config", testRootDir) glog.Infof("Using %s as root dir for kubelet #1", testRootDir) - kcfg := kubeletapp.SimpleKubelet(cl, &fakeDocker1, machineList[0], testRootDir, firstManifestURL, "127.0.0.1", 10250, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, configFilePath, nil, kubelet.FakeOS{}) + kcfg := kubeletapp.SimpleKubelet(cl, &fakeDocker1, machineList[0], testRootDir, firstManifestURL, "127.0.0.1", 10250, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, configFilePath, nil, kubecontainer.FakeOS{}) kubeletapp.RunKubelet(kcfg, nil) // Kubelet (machine) // Create a second kubelet so that the guestbook example's two redis slaves both // have a place they can schedule. testRootDir = makeTempDirOrDie("kubelet_integ_2.", "") glog.Infof("Using %s as root dir for kubelet #2", testRootDir) - kcfg = kubeletapp.SimpleKubelet(cl, &fakeDocker2, machineList[1], testRootDir, secondManifestURL, "127.0.0.1", 10251, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, "", nil, kubelet.FakeOS{}) + kcfg = kubeletapp.SimpleKubelet(cl, &fakeDocker2, machineList[1], testRootDir, secondManifestURL, "127.0.0.1", 10251, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, "", nil, kubecontainer.FakeOS{}) kubeletapp.RunKubelet(kcfg, nil) return apiServer.URL, configFilePath } diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 83180540452..b41848db012 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -40,6 +40,7 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config" + kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/network" "github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports" @@ -375,7 +376,7 @@ func SimpleKubelet(client *client.Client, cadvisorInterface cadvisor.Interface, configFilePath string, cloud cloudprovider.Interface, - osInterface kubelet.OSInterface) *KubeletConfig { + osInterface kubecontainer.OSInterface) *KubeletConfig { imageGCPolicy := kubelet.ImageGCPolicy{ HighThresholdPercent: 90, @@ -436,7 +437,7 @@ func RunKubelet(kcfg *KubeletConfig, builder KubeletBuilder) { builder = createAndInitKubelet } if kcfg.OSInterface == nil { - kcfg.OSInterface = kubelet.RealOS{} + kcfg.OSInterface = kubecontainer.RealOS{} } k, podCfg, err := builder(kcfg) if err != nil { @@ -534,7 +535,7 @@ type KubeletConfig struct { Cloud cloudprovider.Interface NodeStatusUpdateFrequency time.Duration ResourceContainer string - OSInterface kubelet.OSInterface + OSInterface kubecontainer.OSInterface } func createAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) { diff --git a/cmd/kubernetes/kubernetes.go b/cmd/kubernetes/kubernetes.go index 9a50165dcb0..edcd2185ffc 100644 --- a/cmd/kubernetes/kubernetes.go +++ b/cmd/kubernetes/kubernetes.go @@ -37,8 +37,8 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/nodecontroller" "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/servicecontroller" "github.com/GoogleCloudPlatform/kubernetes/pkg/controller" - "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor" + kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" "github.com/GoogleCloudPlatform/kubernetes/pkg/master" "github.com/GoogleCloudPlatform/kubernetes/pkg/service" @@ -159,7 +159,7 @@ func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr net.IP if err != nil { glog.Fatalf("Failed to create cAdvisor: %v", err) } - kcfg := kubeletapp.SimpleKubelet(cl, dockerClient, machineList[0], "/tmp/kubernetes", "", "127.0.0.1", 10250, *masterServiceNamespace, kubeletapp.ProbeVolumePlugins(), nil, cadvisorInterface, "", nil, kubelet.RealOS{}) + kcfg := kubeletapp.SimpleKubelet(cl, dockerClient, machineList[0], "/tmp/kubernetes", "", "127.0.0.1", 10250, *masterServiceNamespace, kubeletapp.ProbeVolumePlugins(), nil, cadvisorInterface, "", nil, kubecontainer.RealOS{}) kubeletapp.RunKubelet(kcfg, nil) } diff --git a/pkg/kubelet/container/os.go b/pkg/kubelet/container/os.go new file mode 100644 index 00000000000..518f23d0e67 --- /dev/null +++ b/pkg/kubelet/container/os.go @@ -0,0 +1,55 @@ +/* +Copyright 2015 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package container + +import ( + "os" +) + +// OSInterface collects system level operations that need to be mocked out +// during tests. +type OSInterface interface { + Mkdir(path string, perm os.FileMode) error + Symlink(oldname string, newname string) error +} + +// RealOS is used to dispatch the real system level operaitons. +type RealOS struct{} + +// MkDir will will call os.Mkdir to create a directory. +func (RealOS) Mkdir(path string, perm os.FileMode) error { + return os.Mkdir(path, perm) +} + +// Symlink will call os.Symlink to create a symbolic link. +func (RealOS) Symlink(oldname string, newname string) error { + return os.Symlink(oldname, newname) +} + +// FakeOS mocks out certain OS calls to avoid perturbing the filesystem +// on the test machine. +type FakeOS struct{} + +// MkDir is a fake call that just returns nil. +func (FakeOS) Mkdir(path string, perm os.FileMode) error { + return nil +} + +// Symlink is a fake call that just returns nil. +func (FakeOS) Symlink(oldname string, newname string) error { + return nil +} diff --git a/pkg/kubelet/dockertools/docker_test.go b/pkg/kubelet/dockertools/docker_test.go index 5cc595a867f..da2d7b3b660 100644 --- a/pkg/kubelet/dockertools/docker_test.go +++ b/pkg/kubelet/dockertools/docker_test.go @@ -392,7 +392,7 @@ func TestIsImagePresent(t *testing.T) { func TestGetRunningContainers(t *testing.T) { fakeDocker := &FakeDockerClient{Errors: make(map[string]error)} fakeRecorder := &record.FakeRecorder{} - containerManager := NewDockerManager(fakeDocker, fakeRecorder, nil, nil, PodInfraContainerImage, 0, 0) + containerManager := NewDockerManager(fakeDocker, fakeRecorder, nil, nil, PodInfraContainerImage, 0, 0, "", kubecontainer.FakeOS{}) tests := []struct { containers map[string]*docker.Container inputIDs []string @@ -657,7 +657,7 @@ func TestFindContainersByPod(t *testing.T) { }, } fakeClient := &FakeDockerClient{} - containerManager := NewDockerManager(fakeClient, &record.FakeRecorder{}, nil, nil, PodInfraContainerImage, 0, 0) + containerManager := NewDockerManager(fakeClient, &record.FakeRecorder{}, nil, nil, PodInfraContainerImage, 0, 0, "", kubecontainer.FakeOS{}) for i, test := range tests { fakeClient.ContainerList = test.containerList fakeClient.ExitedContainerList = test.exitedContainerList diff --git a/pkg/kubelet/dockertools/manager.go b/pkg/kubelet/dockertools/manager.go index 0aee6c01ea3..c916d766827 100644 --- a/pkg/kubelet/dockertools/manager.go +++ b/pkg/kubelet/dockertools/manager.go @@ -56,6 +56,7 @@ type DockerManager struct { recorder record.EventRecorder readinessManager *kubecontainer.ReadinessManager containerRefManager *kubecontainer.RefManager + os kubecontainer.OSInterface // TODO(yifan): PodInfraContainerImage can be unexported once // we move createPodInfraContainer into dockertools. @@ -74,6 +75,12 @@ type DockerManager struct { // use the concrete type so that we can record the pull failure and eliminate // the image checking in GetPodStatus(). Puller DockerPuller + + // Root of the Docker runtime. + dockerRoot string + + // Directory of container logs. + containerLogsDir string } func NewDockerManager( @@ -83,16 +90,51 @@ func NewDockerManager( containerRefManager *kubecontainer.RefManager, podInfraContainerImage string, qps float32, - burst int) *DockerManager { + burst int, + containerLogsDir string, + osInterface kubecontainer.OSInterface) *DockerManager { + // Work out the location of the Docker runtime, defaulting to /var/lib/docker + // if there are any problems. + dockerRoot := "/var/lib/docker" + dockerInfo, err := client.Info() + if err != nil { + glog.Errorf("Failed to execute Info() call to the Docker client: %v", err) + glog.Warningf("Using fallback default of /var/lib/docker for location of Docker runtime") + } else { + driverStatus := dockerInfo.Get("DriverStatus") + // The DriverStatus is a*string* which represents a list of list of strings (pairs) e.g. + // DriverStatus=[["Root Dir","/var/lib/docker/aufs"],["Backing Filesystem","extfs"],["Dirs","279"]] + // Strip out the square brakcets and quotes. + s := strings.Replace(driverStatus, "[", "", -1) + s = strings.Replace(s, "]", "", -1) + s = strings.Replace(s, `"`, "", -1) + // Separate by commas. + ss := strings.Split(s, ",") + // Search for the Root Dir string + for i, k := range ss { + if k == "Root Dir" && i+1 < len(ss) { + // Discard the /aufs suffix. + dockerRoot, _ = path.Split(ss[i+1]) + // Trim the last slash. + dockerRoot = strings.TrimSuffix(dockerRoot, "/") + glog.Infof("Setting dockerRoot to %s", dockerRoot) + } + + } + } + reasonCache := stringCache{cache: lru.New(maxReasonCacheEntries)} return &DockerManager{ - client: client, - recorder: recorder, - readinessManager: readinessManager, - containerRefManager: containerRefManager, + client: client, + recorder: recorder, + readinessManager: readinessManager, + containerRefManager: containerRefManager, + os: osInterface, PodInfraContainerImage: podInfraContainerImage, reasonCache: reasonCache, Puller: newDockerPuller(client, qps, burst), + dockerRoot: dockerRoot, + containerLogsDir: containerLogsDir, } } @@ -936,6 +978,17 @@ func (dm *DockerManager) RunContainer(pod *api.Pod, container *api.Container, ge return DockerID(""), fmt.Errorf("failed to call event handler: %v", handlerErr) } } + + // Create a symbolic link to the Docker container log file using a name which captures the + // full pod name, the container name and the Docker container ID. Cluster level logging will + // capture these symbolic filenames which can be used for search terms in Elasticsearch or for + // labels for Cloud Logging. + podFullName := kubecontainer.GetPodFullName(pod) + containerLogFile := path.Join(dm.dockerRoot, "containers", id, fmt.Sprintf("%s-json.log", id)) + symlinkFile := path.Join(dm.containerLogsDir, fmt.Sprintf("%s-%s-%s.log", podFullName, container.Name, id)) + if err = dm.os.Symlink(containerLogFile, symlinkFile); err != nil { + glog.Errorf("Failed to create symbolic link to the log file of pod %q container %q: %v", podFullName, container.Name, err) + } return DockerID(id), err } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 4e4f7f04f6a..ba6aef6b8ea 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -69,6 +69,9 @@ const ( // nodeStatusUpdateRetry specifies how many times kubelet retries when posting node status failed. nodeStatusUpdateRetry = 5 + + // Location of container logs. + containerLogsDir = "/var/log/containers" ) var ( @@ -95,40 +98,6 @@ type SourcesReadyFn func() bool type volumeMap map[string]volume.Volume -// OSInterface collects system level operations that need to be mocked out -// during tests. -type OSInterface interface { - Mkdir(path string, perm os.FileMode) error - Symlink(oldname string, newname string) error -} - -// RealOS is used to dispatch the real system level operaitons. -type RealOS struct{} - -// MkDir will will call os.Mkdir to create a directory. -func (RealOS) Mkdir(path string, perm os.FileMode) error { - return os.Mkdir(path, perm) -} - -// Symlink will call os.Symlink to create a symbolic link. -func (RealOS) Symlink(oldname string, newname string) error { - return os.Symlink(oldname, newname) -} - -// FakeOS mocks out certain OS calls to avoid perturbing the filesystem -// on the test machine. -type FakeOS struct{} - -// MkDir is a fake call that just returns nil. -func (FakeOS) Mkdir(path string, perm os.FileMode) error { - return nil -} - -// Symlink is a fake call that just returns nil. -func (FakeOS) Symlink(oldname string, newname string) error { - return nil -} - // New creates a new Kubelet for use in main func NewMainKubelet( hostname string, @@ -154,7 +123,7 @@ func NewMainKubelet( cloud cloudprovider.Interface, nodeStatusUpdateFrequency time.Duration, resourceContainer string, - osInterface OSInterface) (*Kubelet, error) { + osInterface kubecontainer.OSInterface) (*Kubelet, error) { if rootDirectory == "" { return nil, fmt.Errorf("invalid root directory %q", rootDirectory) } @@ -178,35 +147,6 @@ func NewMainKubelet( if !dockerUp { return nil, fmt.Errorf("timed out waiting for Docker to come up") } - // Work out the location of the Docker runtime, defaulting to /var/lib/docker - // if there are any problems. - dockerRoot := "/var/lib/docker" - dockerInfo, err := dockerClient.Info() - if err != nil { - glog.Errorf("Failed to execute Info() call to the Docker client: %v", err) - glog.Warningf("Using fallback default of /var/lib/docker for location of Docker runtime") - } else { - driverStatus := dockerInfo.Get("DriverStatus") - // The DriverStatus is a*string* which represents a list of list of strings (pairs) e.g. - // DriverStatus=[["Root Dir","/var/lib/docker/aufs"],["Backing Filesystem","extfs"],["Dirs","279"]] - // Strip out the square brakcets and quotes. - s := strings.Replace(driverStatus, "[", "", -1) - s = strings.Replace(s, "]", "", -1) - s = strings.Replace(s, `"`, "", -1) - // Separate by commas. - ss := strings.Split(s, ",") - // Search for the Root Dir string - for i, k := range ss { - if k == "Root Dir" && i+1 < len(ss) { - // Discard the /aufs suffix. - dockerRoot, _ = path.Split(ss[i+1]) - // Trim the last slash. - dockerRoot = strings.TrimSuffix(dockerRoot, "/") - glog.Infof("Setting dockerRoot to %s", dockerRoot) - } - - } - } serviceStore := cache.NewStore(cache.MetaNamespaceKeyFunc) if kubeClient != nil { @@ -269,7 +209,9 @@ func NewMainKubelet( containerRefManager, podInfraContainerImage, pullQPS, - pullBurst) + pullBurst, + containerLogsDir, + osInterface) volumeManager := newVolumeManager() @@ -302,7 +244,6 @@ func NewMainKubelet( nodeStatusUpdateFrequency: nodeStatusUpdateFrequency, resourceContainer: resourceContainer, os: osInterface, - dockerRoot: dockerRoot, } klet.podManager = newBasicPodManager(klet.kubeClient) @@ -330,10 +271,10 @@ func NewMainKubelet( } else { klet.networkPlugin = plug } - // If the /var/log/containers directory does not exist, create it. - if _, err := os.Stat("/var/log/containers"); err != nil { - if err := osInterface.Mkdir("/var/log/containers", 0755); err != nil { - glog.Errorf("Failed to create directory /var/log/containers: %v", err) + // If the container logs directory does not exist, create it. + if _, err := os.Stat(containerLogsDir); err != nil { + if err := osInterface.Mkdir(containerLogsDir, 0755); err != nil { + glog.Errorf("Failed to create directory %q: %v", containerLogsDir, err) } } @@ -453,8 +394,7 @@ type Kubelet struct { // Name must be absolute. resourceContainer string - os OSInterface - dockerRoot string + os kubecontainer.OSInterface } // getRootDir returns the full path to the directory under which kubelet can @@ -1014,21 +954,6 @@ func (kl *Kubelet) pullImageAndRunContainer(pod *api.Pod, container *api.Contain glog.Errorf("Error running pod %q container %q: %v", podFullName, container.Name, err) return "", err } - // Create a symbolic link to the Docker container log file using a name which captures the - // full pod name, the container name and the Docker container ID. Cluster level logging will - // capture these symbolic filenames which can be used for search terms in Elasticsearch or for - // labels for Cloud Logging. - // If for any reason kl.dockerRoot is not set, default to /var/lib/docker - dockerRoot := kl.dockerRoot - if kl.dockerRoot == "" { - dockerRoot = "/var/lib/docker" - glog.Errorf("dockerRoot field not set in the Kubelet configuration") - } - containerLogFile := fmt.Sprintf("%s/containers/%s/%s-json.log", dockerRoot, containerID, containerID) - symlinkFile := fmt.Sprintf("/var/log/containers/%s-%s-%s.log", podFullName, container.Name, containerID) - if err = kl.os.Symlink(containerLogFile, symlinkFile); err != nil { - glog.Errorf("Failed to create symbolic link to the log file of pod %q container %q: %v", podFullName, container.Name, err) - } return containerID, nil } diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 45e4358487b..c349387b502 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -75,7 +75,7 @@ func newTestKubelet(t *testing.T) *TestKubelet { kubelet := &Kubelet{} kubelet.dockerClient = fakeDocker kubelet.kubeClient = fakeKubeClient - kubelet.os = FakeOS{} + kubelet.os = kubecontainer.FakeOS{} kubelet.hostname = "testnode" kubelet.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", network.NewFakeHost(nil)) @@ -103,7 +103,7 @@ func newTestKubelet(t *testing.T) *TestKubelet { podManager, fakeMirrorClient := newFakePodManager() kubelet.podManager = podManager kubelet.containerRefManager = kubecontainer.NewRefManager() - kubelet.containerManager = dockertools.NewDockerManager(fakeDocker, fakeRecorder, kubelet.readinessManager, kubelet.containerRefManager, dockertools.PodInfraContainerImage, 0, 0) + kubelet.containerManager = dockertools.NewDockerManager(fakeDocker, fakeRecorder, kubelet.readinessManager, kubelet.containerRefManager, dockertools.PodInfraContainerImage, 0, 0, "", kubelet.os) kubelet.runtimeCache = kubecontainer.NewFakeRuntimeCache(kubelet.containerManager) kubelet.podWorkers = newPodWorkers( kubelet.runtimeCache, diff --git a/pkg/kubelet/pod_workers_test.go b/pkg/kubelet/pod_workers_test.go index 9f11c3880cf..24ac7ac8678 100644 --- a/pkg/kubelet/pod_workers_test.go +++ b/pkg/kubelet/pod_workers_test.go @@ -40,7 +40,7 @@ func newPod(uid, name string) *api.Pod { func createPodWorkers() (*podWorkers, map[types.UID][]string) { fakeDocker := &dockertools.FakeDockerClient{} fakeRecorder := &record.FakeRecorder{} - dockerManager := dockertools.NewDockerManager(fakeDocker, fakeRecorder, nil, nil, dockertools.PodInfraContainerImage, 0, 0) + dockerManager := dockertools.NewDockerManager(fakeDocker, fakeRecorder, nil, nil, dockertools.PodInfraContainerImage, 0, 0, "", kubecontainer.FakeOS{}) fakeRuntimeCache := kubecontainer.NewFakeRuntimeCache(dockerManager) lock := sync.Mutex{} diff --git a/pkg/kubelet/runonce_test.go b/pkg/kubelet/runonce_test.go index 7489ad96f57..544885d2ef4 100644 --- a/pkg/kubelet/runonce_test.go +++ b/pkg/kubelet/runonce_test.go @@ -86,7 +86,7 @@ func TestRunOnce(t *testing.T) { containerRefManager: kubecontainer.NewRefManager(), readinessManager: kubecontainer.NewReadinessManager(), podManager: podManager, - os: FakeOS{}, + os: kubecontainer.FakeOS{}, volumeManager: newVolumeManager(), } @@ -154,7 +154,9 @@ func TestRunOnce(t *testing.T) { kb.containerRefManager, dockertools.PodInfraContainerImage, 0, - 0) + 0, + "", + kubecontainer.FakeOS{}) kb.containerManager.Puller = &dockertools.FakeDockerPuller{} pods := []*api.Pod{