kubernetes: modernise image cache builds

This updates the build of the two image caches to use the `pkg/package.mk`
infrastructure, albeit in a slightly (ok, very) atypical way.

In order to share the bulk of the build code (including the `Dockerfile` and
the `Makefile` machinery to download the images) we arrange for the necessary
bits to be copied at build time into distinct subdirectories and for the
`pkg/package.mk` to be aware of this possibility.

Since pkg/package.mk is only set up to build a single package we use a single
`image-cache/Makefile` to drive the whole process and recurse into
`Makefile.pkg` to build individual packages.

One particular subtlety is that the package hash is based on the `image-cache`
directory (which is in `git`) rather than the generated subdirectories (which
are not in `git`). Since all the generators (and their inputs) are in the
`image-cache` directory this is what we want. This means that the two images
are given the same tag, but this is deliberate and desirable.

The generated directories are completely temporary to avoid picking up stale
versions of images when versions are updated. Images are hardlinked into place.

The images are moved to the linuxkitprojects org. Using a dev tag for now, will
update once everything is in place.

Also use "tag" rather than "build" where appropriate in the Makefile.

There is no point in the .dockerignore now, but add a .gitignore.

Signed-off-by: Ian Campbell <ijc@docker.com>

pkg/package.mk

@@ -2,6 +2,12 @@
 default: push
 
 ORG?=linuxkit
+SOURCE ?= .
+
+# Hash is of $(CURDIR) not $(CURDIR)$(SOURCE) to allow autogenerated
+# source subdirectories (which would not be covered by ls-tree, but
+# the code which autogenerates should be in $(CURDIR) so that is what
+# we want to use.
 ifeq ($(HASH),)
 HASH_COMMIT?=HEAD # Setting this is only really useful with the show-tag target
 HASH?=$(shell git ls-tree --full-tree $(HASH_COMMIT) -- $(CURDIR) | awk '{print $$3}')
@@ -44,10 +50,10 @@ show-tag:
 	@echo $(TAG)
 
 tag: $(BASE_DEPS) $(DEPS)
-	docker pull $(TAG) || docker build $(LABELS) $(NET_OPT) -t $(TAG) .
+	docker pull $(TAG) || docker build $(LABELS) $(NET_OPT) -t $(TAG) $(SOURCE)
 
 forcetag: $(BASE_DEPS) $(DEPS)
-	docker build $(LABELS) $(NET_OPT) -t $(TAG) .
+	docker build $(LABELS) $(NET_OPT) -t $(TAG) $(SOURCE)
 
 push: tag
 ifneq ($(DIRTY),)

projects/kubernetes/Makefile

@@ -1,12 +1,14 @@
-all: build-container-images build-vm-images
+all: tag-container-images build-vm-images
 
-build-container-images:
-	make -C kubernetes tag
+tag-container-images:
+	$(MAKE) -C kubernetes tag
 
-push-container-images: cache-images
-	make -C kubernetes push
-	docker image push linuxkit/kubernetes:latest-image-cache-common
-	docker image push linuxkit/kubernetes:latest-image-cache-control-plane
+tag-cache-images:
+	$(MAKE) -C image-cache tag
+
+push-container-images:
+	$(MAKE) -C kubernetes push
+	$(MAKE) -C image-cache push
 
 build-vm-images: kube-master-initrd.img kube-node-initrd.img
 
@@ -18,35 +20,5 @@ kube-node-initrd.img: kube-node.yml
 
 clean:
 	rm -f -r \
-	  kube-*-kernel kube-*-cmdline kube-*-state kube-*-initrd.img \
-	  image-cache/common image-cache/control-plane
-
-COMMON_IMAGES := \
-  kube-proxy-amd64:v1.6.1 \
-  k8s-dns-sidecar-amd64:1.14.1 \
-  k8s-dns-kube-dns-amd64:1.14.1 \
-  k8s-dns-dnsmasq-nanny-amd64:1.14.1 \
-  pause-amd64:3.0
-
-CONTROL_PLANE_IMAGES := \
-  kube-apiserver-amd64:v1.6.1 \
-  kube-controller-manager-amd64:v1.6.1 \
-  kube-scheduler-amd64:v1.6.1 \
-  etcd-amd64:3.0.17
-
-image-cache/%.tar:
-	mkdir -p $(dir $@)
-	docker image pull gcr.io/google_containers/$(shell basename $@ .tar)
-	docker image save -o $@ gcr.io/google_containers/$(shell basename $@ .tar)
-
-cache-images:
-	for image in $(COMMON_IMAGES) ; \
-	  do $(MAKE) "image-cache/common/$${image}.tar" \
-	  ; done
-	cp image-cache/Dockerfile image-cache/.dockerignore image-cache/common
-	docker image build -t linuxkit/kubernetes:latest-image-cache-common image-cache/common
-	for image in $(CONTROL_PLANE_IMAGES) ; \
-	  do $(MAKE) "image-cache/control-plane/$${image}.tar" \
-	  ; done
-	cp image-cache/Dockerfile image-cache/.dockerignore image-cache/control-plane
-	docker image build -t linuxkit/kubernetes:latest-image-cache-control-plane image-cache/control-plane
+	  kube-*-kernel kube-*-cmdline kube-*-state kube-*-initrd.img
+	$(MAKE) -C image-cache clean

projects/kubernetes/image-cache/.dockerignore

@@ -1 +0,0 @@
-Dockerfile

projects/kubernetes/image-cache/.gitignore

@@ -0,0 +1 @@
+dl/

projects/kubernetes/image-cache/Makefile

@@ -0,0 +1,45 @@
+default: push
+
+COMMON_IMAGES := \
+  kube-proxy-amd64\:v1.6.1 \
+  k8s-dns-sidecar-amd64\:1.14.1 \
+  k8s-dns-kube-dns-amd64\:1.14.1 \
+  k8s-dns-dnsmasq-nanny-amd64\:1.14.1 \
+  pause-amd64\:3.0
+
+CONTROL_PLANE_IMAGES := \
+  kube-apiserver-amd64\:v1.6.1 \
+  kube-controller-manager-amd64\:v1.6.1 \
+  kube-scheduler-amd64\:v1.6.1 \
+  etcd-amd64\:3.0.17
+
+dl/%.tar:
+	mkdir -p $(dir $@)
+	docker image pull gcr.io/google_containers/$(shell basename $@ .tar)
+	docker image save -o $@ gcr.io/google_containers/$(shell basename $@ .tar)
+
+%-pkg:
+	set -e ; \
+	builddir=$$(mktemp -d $(CACHE).XXXXXX) ; \
+	trap 'rm -rf $${builddir}' EXIT ; \
+	ln $(IMAGES) $${builddir} ; \
+	$(MAKE) -f Makefile.pkg BUILDDIR=$${builddir} CACHE=$(CACHE) $*
+
+.PHONY: tag-common push-common
+tag-common push-common: %-common: $(patsubst %,dl/%.tar,$(COMMON_IMAGES))
+	$(MAKE) CACHE=common IMAGES="$^" $*-pkg
+
+.PHONY: tag-control-plane push-control-plane
+tag-control-plane push-control-plane: %-control-plane: $(patsubst %,dl/%.tar,$(CONTROL_PLANE_IMAGES))
+	$(MAKE) CACHE=control-plane IMAGES="$^" $*-pkg
+
+.PHONY: tag push
+tag: tag-common tag-control-plane
+push: push-common push-control-plane
+
+.PHONY: dl
+dl: $(patsubst %,dl/%.tar,$(COMMON_IMAGES) $(CONTROL_PLANE_IMAGES))
+
+.PHONY: clean
+clean:
+	rm -rf dl

projects/kubernetes/image-cache/Makefile.pkg

@@ -0,0 +1,10 @@
+ORG?=linuxkitprojects
+IMAGE=kubernetes-image-cache-$(CACHE)
+NOTRUST=1
+SOURCE=$(BUILDDIR)
+DEPS=$(BUILDDIR)/Dockerfile
+
+$(BUILDDIR)/Dockerfile: Dockerfile
+	cp $< $@
+
+include ../../../pkg/package.mk

projects/kubernetes/kube-master.yml

@@ -56,9 +56,9 @@ services:
     rootfsPropagation: shared
     command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
   - name: kubernetes-image-cache-common
-    image: linuxkit/kubernetes:latest-image-cache-common
+    image: linuxkitprojects/kubernetes-image-cache-common:dev
   - name: kubernetes-image-cache-control-plane
-    image: linuxkit/kubernetes:latest-image-cache-control-plane
+    image: linuxkitprojects/kubernetes-image-cache-control-plane:dev
   - name: kubelet
     image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14
 files:

projects/kubernetes/kube-node.yml

@@ -56,7 +56,7 @@ services:
     rootfsPropagation: shared
     command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
   - name: kubernetes-image-cache-common
-    image: linuxkit/kubernetes:latest-image-cache-common
+    image: linuxkitprojects/kubernetes-image-cache-common:dev
   - name: kubelet
     image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14
 files: