Merge pull request #84728 from notpad/kubemark

Migrate Kubemark to distroless

build/root/WORKSPACE

@@ -73,11 +73,11 @@ container_repositories()
 load("@io_bazel_rules_docker//container:container.bzl", "container_pull")
 
 container_pull(
-    name = "debian_jessie",
+    name = "distroless_base",
-    digest = "sha256:e25703ee6ab5b2fac31510323d959cdae31eebdf48e88891c549e55b25ad7e94",
+    digest = "sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5",
-    registry = "index.docker.io",
+    registry = "gcr.io",
-    repository = "library/debian",
+    repository = "distroless/base",
-    tag = "jessie",  # ignored when digest provided, but kept here for documentation.
+    tag = "latest",  # ignored when digest provided, but kept here for documentation.
 )
 
 load("//build:workspace.bzl", "release_dependencies")

cluster/images/kubemark/BUILD

@@ -4,7 +4,7 @@ load("@io_bazel_rules_docker//container:container.bzl", "container_image", "cont
 
 container_image(
     name = "image",
-    base = "@debian_jessie//image",
+    base = "@distroless_base//image",
     entrypoint = ["/kubemark"],
     files = ["//cmd/kubemark"],
     stamp = True,

cluster/images/kubemark/Dockerfile

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# The line below points to debian:jessie as of 2019-10-23. The SHA should be
+# The line below points to distroless/base as of 2019-11-15. The SHA should be
-# kept in sycn with debian_jessie definition in the WORKSPACE file.
+# kept in sycn with distroless_base definition in the WORKSPACE file.
-FROM debian@sha256:e25703ee6ab5b2fac31510323d959cdae31eebdf48e88891c549e55b25ad7e94
+FROM gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5
 
 COPY kubemark /kubemark

pkg/kubemark/hollow_kubelet.go

@@ -111,7 +111,7 @@ func NewHollowKubelet(
 		VolumePlugins:      volumePlugins(),
 		TLSOptions:         nil,
 		OOMAdjuster:        oom.NewFakeOOMAdjuster(),
-		Mounter:            mount.New("" /* default mount path */),
+		Mounter:            &mount.FakeMounter{},
 		Subpather:          &subpath.FakeSubpath{},
 		HostUtil:           hostutil.NewFakeHostUtil(nil),
 	}

test/kubemark/iks/startup.sh

@@ -235,13 +235,18 @@ EOF
   fi
   proxy_mem_per_node=50
   proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES))
+  hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
+  hollow_kubelet_params=${hollow_kubelet_params%?}
+  hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
+  hollow_proxy_params=${hollow_proxy_params%?}
+
   sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s'{{kubemark_image_registry}}'${KUBEMARK_IMAGE_REGISTRY}${KUBE_NAMESPACE}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s/{{kubemark_image_tag}}/${KUBEMARK_IMAGE_TAG}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s/{{hollow_kubelet_params}}/${HOLLOW_KUBELET_TEST_ARGS:-}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
+  sed -i'' -e "s/{{hollow_kubelet_params}}/${hollow_kubelet_params}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s/{{hollow_proxy_params}}/${HOLLOW_PROXY_TEST_ARGS:-}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
+  sed -i'' -e "s/{{hollow_proxy_params}}/${hollow_proxy_params}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s'{{kubemark_mig_config}}'${KUBEMARK_MIG_CONFIG:-}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"
 

test/kubemark/resources/hollow-node_template.yaml

@@ -50,10 +50,16 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        command:
+        command: [
-        - /bin/sh
+          "/kubemark",
-        - -c
+          "--morph=kubelet",
-        - /kubemark --morph=kubelet --name=$(NODE_NAME) {{hollow_kubelet_params}} --kubeconfig=/kubeconfig/kubelet.kubeconfig $(CONTENT_TYPE) --alsologtostderr 1>>/var/log/kubelet-$(NODE_NAME).log 2>&1
+          "--name=$(NODE_NAME)",
+          "--kubeconfig=/kubeconfig/kubelet.kubeconfig",
+          "$(CONTENT_TYPE)",
+          "--log-file=/var/log/kubelet-$(NODE_NAME).log",
+          "--logtostderr=false",
+          {{hollow_kubelet_params}}
+        ]
         volumeMounts:
         - name: kubeconfig-volume
           mountPath: /kubeconfig
@@ -78,10 +84,16 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        command:
+        command: [
-        - /bin/sh
+          "/kubemark",
-        - -c
+          "--morph=proxy",
-        - /kubemark --morph=proxy --name=$(NODE_NAME) {{hollow_proxy_params}} --kubeconfig=/kubeconfig/kubeproxy.kubeconfig $(CONTENT_TYPE) --alsologtostderr 1>>/var/log/kubeproxy-$(NODE_NAME).log 2>&1
+          "--name=$(NODE_NAME)",
+          "--kubeconfig=/kubeconfig/kubeproxy.kubeconfig",
+          "$(CONTENT_TYPE)",
+          "--log-file=/var/log/kubeproxy-$(NODE_NAME).log",
+          "--logtostderr=false",
+          {{hollow_proxy_params}}
+        ]
         volumeMounts:
         - name: kubeconfig-volume
           mountPath: /kubeconfig

test/kubemark/start-kubemark.sh

@@ -152,6 +152,10 @@ function create-kube-hollow-node-resources {
   proxy_cpu=${KUBEMARK_HOLLOW_PROXY_MILLICPU:-$proxy_cpu}
   proxy_mem_per_node=${KUBEMARK_HOLLOW_PROXY_MEM_PER_NODE_KB:-50}
   proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES))
+  hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
+  hollow_kubelet_params=${hollow_kubelet_params%?}
+  hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
+  hollow_proxy_params=${hollow_proxy_params%?}
 
   sed -i'' -e "s@{{hollow_kubelet_millicpu}}@${KUBEMARK_HOLLOW_KUBELET_MILLICPU:-40}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s@{{hollow_kubelet_mem_Ki}}@${KUBEMARK_HOLLOW_KUBELET_MEM_KB:-$((100*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
@@ -162,8 +166,8 @@ function create-kube-hollow-node-resources {
   sed -i'' -e "s@{{kubemark_image_registry}}@${KUBEMARK_IMAGE_REGISTRY}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s@{{kubemark_image_tag}}@${KUBEMARK_IMAGE_TAG}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s@{{master_ip}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{hollow_kubelet_params}}@${HOLLOW_KUBELET_TEST_ARGS}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
+  sed -i'' -e "s@{{hollow_kubelet_params}}@${hollow_kubelet_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{hollow_proxy_params}}@${HOLLOW_PROXY_TEST_ARGS}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
+  sed -i'' -e "s@{{hollow_proxy_params}}@${hollow_proxy_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s@{{kubemark_mig_config}}@${KUBEMARK_MIG_CONFIG:-}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"