Merge pull request #4117 from wojtek-t/validate_docker_version

Check Docker version in Kubelet /healthz handler

cmd/kubernetes/kubernetes.go

@@ -33,6 +33,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	nodeControllerPkg "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/controller"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/controller"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
 	kubeletServer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/server"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports"
@@ -139,7 +140,7 @@ func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr net.IP
 	runScheduler(cl)
 	runControllerManager(machineList, cl, *nodeMilliCPU, *nodeMemory)
 
-	dockerClient := util.ConnectToDockerOrDie(*dockerEndpoint)
+	dockerClient := dockertools.ConnectToDockerOrDie(*dockerEndpoint)
 	kubeletServer.SimpleRunKubelet(cl, nil, dockerClient, machineList[0], "/tmp/kubernetes", "", "127.0.0.1", 10250, *masterServiceNamespace, kubeletServer.ProbeVolumePlugins())
 }
 

hack/test-cmd.sh

@@ -55,6 +55,7 @@ kube::log::status "Starting kubelet in masterless mode"
 "${KUBE_OUTPUT_HOSTBIN}/kubelet" \
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
+  --docker_endpoint="fake://" \
   --address="127.0.0.1" \
   --port="$KUBELET_PORT" 1>&2 &
 KUBELET_PID=$!
@@ -65,6 +66,7 @@ kube::log::status "Starting kubelet in masterful mode"
 "${KUBE_OUTPUT_HOSTBIN}/kubelet" \
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
+  --docker_endpoint="fake://" \
   --etcd_servers="http://${ETCD_HOST}:${ETCD_PORT}" \
   --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \

pkg/kubelet/dockertools/docker.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"os/exec"
 	"strconv"
 	"strings"
@@ -104,7 +105,7 @@ type dockerContainerCommandRunner struct {
 var dockerAPIVersionWithExec = []uint{1, 15}
 
 // Returns the major and minor version numbers of docker server.
-func (d *dockerContainerCommandRunner) getDockerServerVersion() ([]uint, error) {
+func (d *dockerContainerCommandRunner) GetDockerServerVersion() ([]uint, error) {
 	env, err := d.client.Version()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get docker server version - %v", err)
@@ -127,7 +128,7 @@ func (d *dockerContainerCommandRunner) getDockerServerVersion() ([]uint, error)
 }
 
 func (d *dockerContainerCommandRunner) nativeExecSupportExists() (bool, error) {
-	version, err := d.getDockerServerVersion()
+	version, err := d.GetDockerServerVersion()
 	if err != nil {
 		return false, err
 	}
@@ -641,6 +642,35 @@ func parseImageName(image string) (string, string) {
 	return image, tag
 }
 
+// Get a docker endpoint, either from the string passed in, or $DOCKER_HOST environment variables
+func getDockerEndpoint(dockerEndpoint string) string {
+	var endpoint string
+	if len(dockerEndpoint) > 0 {
+		endpoint = dockerEndpoint
+	} else if len(os.Getenv("DOCKER_HOST")) > 0 {
+		endpoint = os.Getenv("DOCKER_HOST")
+	} else {
+		endpoint = "unix:///var/run/docker.sock"
+	}
+	glog.Infof("Connecting to docker on %s", endpoint)
+
+	return endpoint
+}
+
+func ConnectToDockerOrDie(dockerEndpoint string) DockerInterface {
+	if dockerEndpoint == "fake://" {
+		return &FakeDockerClient{
+			VersionInfo: []string{"apiVersion=1.16"},
+		}
+	}
+	client, err := docker.NewClient(getDockerEndpoint(dockerEndpoint))
+	if err != nil {
+		glog.Fatal("Couldn't connect to docker.")
+	}
+	return client
+}
+
 type ContainerCommandRunner interface {
 	RunInContainer(containerID string, cmd []string) ([]byte, error)
+	GetDockerServerVersion() ([]uint, error)
 }

pkg/kubelet/dockertools/docker_test.go

@@ -123,7 +123,7 @@ func TestContainerManifestNaming(t *testing.T) {
 func TestGetDockerServerVersion(t *testing.T) {
 	fakeDocker := &FakeDockerClient{VersionInfo: docker.Env{"Client version=1.2", "Server version=1.1.3", "Server API version=1.15"}}
 	runner := dockerContainerCommandRunner{fakeDocker}
-	version, err := runner.getDockerServerVersion()
+	version, err := runner.GetDockerServerVersion()
 	if err != nil {
 		t.Errorf("got error while getting docker server version - %s", err)
 	}

pkg/kubelet/kubelet.go

@@ -1380,6 +1380,15 @@ func (kl *Kubelet) syncLoop(updates <-chan PodUpdate, handler SyncHandler) {
 	}
 }
 
+// Returns Docker version for this Kubelet.
+func (kl *Kubelet) GetDockerVersion() ([]uint, error) {
+	if kl.dockerClient == nil {
+		return nil, fmt.Errorf("no Docker client")
+	}
+	dockerRunner := dockertools.NewDockerContainerCommandRunner(kl.dockerClient)
+	return dockerRunner.GetDockerServerVersion()
+}
+
 // GetKubeletContainerLogs returns logs from the container
 // The second parameter of GetPodStatus and FindPodContainer methods represents pod UUID, which is allowed to be blank
 // TODO: this method is returning logs of random container attempts, when it should be returning the most recent attempt

pkg/kubelet/kubelet_test.go

@@ -1367,6 +1367,10 @@ func (f *fakeContainerCommandRunner) RunInContainer(id string, cmd []string) ([]
 	return []byte{}, f.E
 }
 
+func (f *fakeContainerCommandRunner) GetDockerServerVersion() ([]uint, error) {
+	return nil, nil
+}
+
 func TestRunInContainerNoSuchPod(t *testing.T) {
 	fakeCommandRunner := fakeContainerCommandRunner{}
 	kubelet, fakeDocker := newTestKubelet(t)

pkg/kubelet/server.go

@@ -31,7 +31,6 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/httplog"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
@@ -64,6 +63,7 @@ func ListenAndServeKubeletServer(host HostInterface, address net.IP, port uint,
 type HostInterface interface {
 	GetContainerInfo(podFullName string, uid types.UID, containerName string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error)
 	GetRootInfo(req *info.ContainerInfoRequest) (*info.ContainerInfo, error)
+	GetDockerVersion() ([]uint, error)
 	GetMachineInfo() (*info.MachineInfo, error)
 	GetBoundPods() ([]api.BoundPod, error)
 	GetPodByName(namespace, name string) (*api.BoundPod, bool)
@@ -88,7 +88,7 @@ func NewServer(host HostInterface, enableDebuggingHandlers bool) Server {
 
 // InstallDefaultHandlers registers the default set of supported HTTP request patterns with the mux.
 func (s *Server) InstallDefaultHandlers() {
-	healthz.InstallHandler(s.mux)
+	s.mux.HandleFunc("/healthz", s.handleHealthz)
 	s.mux.HandleFunc("/podInfo", s.handlePodInfoOld)
 	s.mux.HandleFunc("/api/v1beta1/podInfo", s.handlePodInfoVersioned)
 	s.mux.HandleFunc("/boundPods", s.handleBoundPods)
@@ -109,6 +109,42 @@ func (s *Server) error(w http.ResponseWriter, err error) {
 	http.Error(w, fmt.Sprintf("Internal Error: %v", err), http.StatusInternalServerError)
 }
 
+func isValidDockerVersion(ver []uint) (bool, string) {
+	minAllowedVersion := []uint{1, 15}
+	for i := 0; i < len(ver) && i < len(minAllowedVersion); i++ {
+		if ver[i] != minAllowedVersion[i] {
+			if ver[i] < minAllowedVersion[i] {
+				versions := make([]string, len(ver))
+				for i, v := range ver {
+					versions[i] = fmt.Sprint(v)
+				}
+				return false, strings.Join(versions, ".")
+			}
+			return true, ""
+		}
+	}
+	return true, ""
+}
+
+// handleHealthz handles /healthz request and checks Docker version
+func (s *Server) handleHealthz(w http.ResponseWriter, req *http.Request) {
+	versions, err := s.host.GetDockerVersion()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("unknown Docker version"))
+		return
+	}
+	valid, version := isValidDockerVersion(versions)
+	if !valid {
+		w.WriteHeader(http.StatusInternalServerError)
+		msg := "Docker version is too old (" + version + ")"
+		w.Write([]byte(msg))
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("ok"))
+}
+
 // handleContainerLogs handles containerLogs request against the Kubelet
 func (s *Server) handleContainerLogs(w http.ResponseWriter, req *http.Request) {
 	defer req.Body.Close()

pkg/kubelet/server/server.go

@@ -203,7 +203,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		CAdvisorPort:            s.CAdvisorPort,
 		EnableServer:            s.EnableServer,
 		EnableDebuggingHandlers: s.EnableDebuggingHandlers,
-		DockerClient:            util.ConnectToDockerOrDie(s.DockerEndpoint),
+		DockerClient:            dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
 		KubeClient:              client,
 		EtcdClient:              kubelet.EtcdClientOrDie(s.EtcdServerList, s.EtcdConfigFile),
 		MasterServiceNamespace:  s.MasterServiceNamespace,

pkg/kubelet/server_test.go

@@ -42,6 +42,7 @@ type fakeKubelet struct {
 	boundPodsFunc     func() ([]api.BoundPod, error)
 	logFunc           func(w http.ResponseWriter, req *http.Request)
 	runFunc           func(podFullName string, uid types.UID, containerName string, cmd []string) ([]byte, error)
+	dockerVersionFunc func() ([]uint, error)
 	containerLogsFunc func(podFullName, containerName, tail string, follow bool, stdout, stderr io.Writer) error
 }
 
@@ -61,6 +62,10 @@ func (fk *fakeKubelet) GetRootInfo(req *info.ContainerInfoRequest) (*info.Contai
 	return fk.rootInfoFunc(req)
 }
 
+func (fk *fakeKubelet) GetDockerVersion() ([]uint, error) {
+	return fk.dockerVersionFunc()
+}
+
 func (fk *fakeKubelet) GetMachineInfo() (*info.MachineInfo, error) {
 	return fk.machineInfoFunc()
 }

pkg/util/node.go

@@ -17,11 +17,9 @@ limitations under the License.
 package util
 
 import (
-	"os"
 	"os/exec"
 	"strings"
 
-	"github.com/fsouza/go-dockerclient"
 	"github.com/golang/glog"
 )
 
@@ -38,26 +36,3 @@ func GetHostname(hostnameOverride string) string {
 	}
 	return strings.TrimSpace(string(hostname))
 }
-
-// Get a docker endpoint, either from the string passed in, or $DOCKER_HOST environment variables
-func GetDockerEndpoint(dockerEndpoint string) string {
-	var endpoint string
-	if len(dockerEndpoint) > 0 {
-		endpoint = dockerEndpoint
-	} else if len(os.Getenv("DOCKER_HOST")) > 0 {
-		endpoint = os.Getenv("DOCKER_HOST")
-	} else {
-		endpoint = "unix:///var/run/docker.sock"
-	}
-	glog.Infof("Connecting to docker on %s", endpoint)
-
-	return endpoint
-}
-
-func ConnectToDockerOrDie(dockerEndpoint string) *docker.Client {
-	client, err := docker.NewClient(GetDockerEndpoint(dockerEndpoint))
-	if err != nil {
-		glog.Fatal("Couldn't connect to docker.")
-	}
-	return client
-}