Merge pull request #59577 from shyamjvs/fix-kubemark-image

Automatic merge from submit-queue (batch tested with PRs 59054, 59515, 59577). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Uniquify kubemark image builds and use new registry format

Solves issues 1 and 3 in https://github.com/kubernetes/kubernetes/issues/59567.

/cc @wojtek-t 

fyi - @thockin @kubernetes/sig-scalability-misc 

```release-note
NONE
```

cluster/images/kubemark/BUILD

@@ -15,7 +15,7 @@ docker_push(
     registry = "$(REGISTRY)",
     repository = "kubemark",
     stamp = True,
-    tag = "latest",
+    tag = "$(IMAGE_TAG)",
     tags = ["manual"],
 )
 

cluster/images/kubemark/Makefile

@@ -20,16 +20,17 @@
 # .. is not the same as:
 #   make REGISTRY=$VAR
 REGISTRY := $(if $(REGISTRY),$(REGISTRY),staging-k8s.gcr.io)
+IMAGE_TAG := $(if $(IMAGE_TAG),$(IMAGE_TAG),latest)
 
 all: gcloudpush
 
 build:
-	docker build --pull -t $(REGISTRY)/kubemark .
+	docker build --pull -t $(REGISTRY)/kubemark:$(IMAGE_TAG) .
 
 gcloudpush: build
-	gcloud docker -- push $(REGISTRY)/kubemark
+	gcloud docker -- push $(REGISTRY)/kubemark:$(IMAGE_TAG)
 
 push: build
-	docker -- push $(REGISTRY)/kubemark
+	docker -- push $(REGISTRY)/kubemark:$(IMAGE_TAG)
 
 .PHONY: all build gcloudpush push

test/kubemark/cloud-provider-config.sh

@@ -17,4 +17,5 @@
 CLOUD_PROVIDER="${CLOUD_PROVIDER:-gce}"
 CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-gcr.io}"
 PROJECT="${PROJECT:-}"
+FULL_REGISTRY="${CONTAINER_REGISTRY}/${PROJECT}"
 KUBEMARK_IMAGE_MAKE_TARGET="${KUBEMARK_IMAGE_MAKE_TARGET:-gcloudpush}"

test/kubemark/resources/hollow-node_template.yaml

@@ -35,7 +35,7 @@ spec:
         emptyDir: {}
       containers:
       - name: hollow-kubelet
-        image: {{registry}}/{{project}}/kubemark:latest
+        image: {{full_registry}}/kubemark:{{kubemark_image_tag}}
         ports:
         - containerPort: 4194
         - containerPort: 10250
@@ -67,7 +67,7 @@ spec:
         securityContext:
           privileged: true
       - name: hollow-proxy
-        image: {{registry}}/{{project}}/kubemark:latest
+        image: {{full_registry}}/kubemark:{{kubemark_image_tag}}
         env:
         - name: CONTENT_TYPE
           valueFrom:

test/kubemark/start-kubemark.sh

@@ -45,6 +45,10 @@ KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
 KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
 RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
 
+# Generate a random 6-digit alphanumeric tag for the kubemark image.
+# Used to uniquify image builds across different invocations of this script.
+KUBEMARK_IMAGE_TAG=$(head /dev/urandom | tr -dc 'a-z0-9' | fold -w 6 | head -n 1)
+
 # Write all environment variables that we need to pass to the kubemark master,
 # locally to the file ${RESOURCE_DIRECTORY}/kubemark-master-env.sh.
 function create-master-environment-file {
@@ -200,7 +204,7 @@ function create-and-upload-hollow-node-image {
   cd "${MAKE_DIR}"
   RETRIES=3
   for attempt in $(seq 1 ${RETRIES}); do
-    if ! REGISTRY="${FULL_REGISTRY}" make "${KUBEMARK_IMAGE_MAKE_TARGET}"; then
+    if ! REGISTRY="${FULL_REGISTRY}" IMAGE_TAG="${KUBEMARK_IMAGE_TAG}" make "${KUBEMARK_IMAGE_MAKE_TARGET}"; then
       if [[ $((attempt)) -eq "${RETRIES}" ]]; then
         echo "${color_red}Make failed. Exiting.${color_norm}"
         exit 1
@@ -221,7 +225,7 @@ function create-and-upload-hollow-node-image {
 function create-and-upload-hollow-node-image-bazel {
   RETRIES=3
   for attempt in $(seq 1 ${RETRIES}); do
-    if ! bazel run //cluster/images/kubemark:push --define REGISTRY="${FULL_REGISTRY}"; then
+    if ! bazel run //cluster/images/kubemark:push --define REGISTRY="${FULL_REGISTRY}" --define IMAGE_TAG="${KUBEMARK_IMAGE_TAG}"; then
       if [[ $((attempt)) -eq "${RETRIES}" ]]; then
         echo "${color_red}Image push failed. Exiting.${color_norm}"
         exit 1
@@ -395,8 +399,8 @@ current-context: kubemark-context")
   proxy_mem=$((100 * 1024 + ${proxy_mem_per_node}*${NUM_NODES}))
   sed -i'' -e "s/{{HOLLOW_PROXY_CPU}}/${proxy_cpu}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s/{{HOLLOW_PROXY_MEM}}/${proxy_mem}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s/{{registry}}/${CONTAINER_REGISTRY}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
+  sed -i'' -e "s'{{full_registry}}'${FULL_REGISTRY}'g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
+  sed -i'' -e "s/{{kubemark_image_tag}}/${KUBEMARK_IMAGE_TAG}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s/{{kubelet_verbosity_level}}/${KUBELET_TEST_LOG_LEVEL}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
   sed -i'' -e "s/{{kubeproxy_verbosity_level}}/${KUBEPROXY_TEST_LOG_LEVEL}/g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
@@ -457,15 +461,6 @@ write-local-kubeconfig
 copy-resource-files-to-master
 start-master-components
 
-# TODO: Simplify. The caller should have to pass a full registry or nothing.
-if [[ -n "${CONTAINER_REGISTRY}" && -n "${PROJECT}" ]]; then
-  FULL_REGISTRY="${CONTAINER_REGISTRY}/${PROJECT}"
-elif [[ -n "${CONTAINER_REGISTRY}" ]]; then
-  FULL_REGISTRY="${CONTAINER_REGISTRY}"
-else
-  FULL_REGISTRY=staging-k8s.gcr.io
-fi
-
 # Setup for hollow-nodes.
 echo ""
 echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}"