From 55f537631a67e10a18c98a7750ab6c646296e746 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 27 Jun 2017 17:08:59 +0100 Subject: [PATCH 01/12] pkg: add target to print the tag $ make --no-print-directory -C pkg/init/ show-tag linuxkit/init:36c56f0664d49c5a6adc1120d1bf5ba6ac30b389 Useful for scripting etc. Signed-off-by: Ian Campbell --- pkg/package.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/package.mk b/pkg/package.mk index 85dedf645..96297159a 100644 --- a/pkg/package.mk +++ b/pkg/package.mk @@ -1,4 +1,4 @@ -.PHONY: tag push +.PHONY: image tag show-tag default: push ORG?=linuxkit @@ -27,6 +27,9 @@ else NET_OPT=--network=none endif +show-tag: + @echo $(ORG)/$(IMAGE):$(TAG) + tag: $(BASE_DEPS) $(DEPS) DOCKER_CONTENT_TRUST=1 docker pull $(ORG)/$(IMAGE):$(TAG) || \ docker build $(NET_OPT) -t $(ORG)/$(IMAGE):$(TAG) . From b6728a85e94b8de73d7e566bacf5efa4e38cc8cb Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 11:56:09 +0100 Subject: [PATCH 02/12] pkg: Simplify dirty handling By setting `DIRTY` to either "-dirty" or "" directly we can simply use `$(HASH)$(DIRTY)` and avoid make adding a space as it does with `+=`. For the push check we now block pushing if `$(DIRTY)` is non-empty. Signed-off-by: Ian Campbell --- pkg/package.mk | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/pkg/package.mk b/pkg/package.mk index 96297159a..1e2971d5c 100644 --- a/pkg/package.mk +++ b/pkg/package.mk @@ -5,18 +5,8 @@ ORG?=linuxkit HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') BASE_DEPS=Dockerfile Makefile -# Add '-dirty' to hash if the repository is not clean. make does not -# concatenate strings without spaces, so we use the documented trick -# of replacing the space with nothing. -DIRTY=$(shell git diff-index --quiet HEAD --; echo $$?) -ifneq ($(DIRTY),0) -HASH+=-dirty -nullstring := -space := $(nullstring) $(nullstring) -TAG=$(subst $(space),,$(HASH)) -else -TAG=$(HASH) -endif +DIRTY=$(shell git diff-index --quiet HEAD -- ) || echo "-dirty") +TAG=$(HASH)$(DIRTY) # Get a release tag, if present RELEASE=$(shell git tag -l --points-at HEAD) @@ -35,7 +25,7 @@ tag: $(BASE_DEPS) $(DEPS) docker build $(NET_OPT) -t $(ORG)/$(IMAGE):$(TAG) . push: tag -ifneq ($(DIRTY),0) +ifneq ($(DIRTY),) $(error Your repository is not clean. Will not push package image.) endif DOCKER_CONTENT_TRUST=1 docker pull $(ORG)/$(IMAGE):$(TAG) || \ From 4a3d5acb8802fd69cc09a8d2bff4ec767274ba7e Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 12:02:11 +0100 Subject: [PATCH 03/12] pkg: Only consider the package directory itself for dirty tracking. Since this is what is used in the hash. Signed-off-by: Ian Campbell --- pkg/package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/package.mk b/pkg/package.mk index 1e2971d5c..025c68600 100644 --- a/pkg/package.mk +++ b/pkg/package.mk @@ -5,7 +5,7 @@ ORG?=linuxkit HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') BASE_DEPS=Dockerfile Makefile -DIRTY=$(shell git diff-index --quiet HEAD -- ) || echo "-dirty") +DIRTY=$(shell git diff-index --quiet HEAD -- ../$(notdir $(CURDIR)) || echo "-dirty") TAG=$(HASH)$(DIRTY) # Get a release tag, if present From 66f9cc134f735efe5822053e368ae944c1bf1e51 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 12:04:10 +0100 Subject: [PATCH 04/12] pkg: Consolidate $(TAG) to be the full tag Saves lots of repetitions of `$(ORG)/$(IMAGE):$(HASH)$(DIRTY)` throughout the file. Signed-off-by: Ian Campbell --- pkg/package.mk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/package.mk b/pkg/package.mk index 025c68600..e169a1353 100644 --- a/pkg/package.mk +++ b/pkg/package.mk @@ -6,7 +6,7 @@ HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') BASE_DEPS=Dockerfile Makefile DIRTY=$(shell git diff-index --quiet HEAD -- ../$(notdir $(CURDIR)) || echo "-dirty") -TAG=$(HASH)$(DIRTY) +TAG=$(ORG)/$(IMAGE):$(HASH)$(DIRTY) # Get a release tag, if present RELEASE=$(shell git tag -l --points-at HEAD) @@ -18,19 +18,19 @@ NET_OPT=--network=none endif show-tag: - @echo $(ORG)/$(IMAGE):$(TAG) + @echo $(TAG) tag: $(BASE_DEPS) $(DEPS) - DOCKER_CONTENT_TRUST=1 docker pull $(ORG)/$(IMAGE):$(TAG) || \ - docker build $(NET_OPT) -t $(ORG)/$(IMAGE):$(TAG) . + DOCKER_CONTENT_TRUST=1 docker pull $(TAG) || \ + docker build $(NET_OPT) -t $(TAG) . push: tag ifneq ($(DIRTY),) $(error Your repository is not clean. Will not push package image.) endif - DOCKER_CONTENT_TRUST=1 docker pull $(ORG)/$(IMAGE):$(TAG) || \ - DOCKER_CONTENT_TRUST=1 docker push $(ORG)/$(IMAGE):$(TAG) + DOCKER_CONTENT_TRUST=1 docker pull $(TAG) || \ + DOCKER_CONTENT_TRUST=1 docker push $(TAG) ifneq ($(RELEASE),) - docker tag $(ORG)/$(IMAGE):$(TAG) $(ORG)/$(IMAGE):$(RELEASE) + docker tag $(TAG) $(ORG)/$(IMAGE):$(RELEASE) DOCKER_CONTENT_TRUST=1 docker push $(ORG)/$(IMAGE):$(RELEASE) endif From 1ea64d33f04f9201a44899385807f3271ee65e1b Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 12:14:03 +0100 Subject: [PATCH 05/12] pkg: use Jessie-era compatible git command for calculating $(HASH) The current rune for $(HASH) returns nothing with the Jessie version of git (2.1.4). Using `--full-tree` works corectly but requires an absolute path (else it produces e.g. "fatal: ../init: '../init' is outside repository"). Both `ls-tree` and `diff-index` are happy with a full absolute path, which simplifies things since we can use `$(CURDIR)` directly. Tested with a dirty `pkg/init` on both Jessie (git 2.1.4) and Stretch (git 2.11.0) with the following command, which produces identical output in both cases: $ for pkg in init containerd ; do make -C pkg/$pkg --no-print-directory show-tag; ( cd pkg/$pkg && make show-tag ); done linuxkit/init:36c56f0664d49c5a6adc1120d1bf5ba6ac30b389-dirty linuxkit/init:36c56f0664d49c5a6adc1120d1bf5ba6ac30b389-dirty linuxkit/containerd:1e3e8f207421de8deac8cedc26a138d6b1661a0d linuxkit/containerd:1e3e8f207421de8deac8cedc26a138d6b1661a0d Signed-off-by: Ian Campbell --- pkg/package.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/package.mk b/pkg/package.mk index e169a1353..6f32c4eb1 100644 --- a/pkg/package.mk +++ b/pkg/package.mk @@ -2,10 +2,10 @@ default: push ORG?=linuxkit -HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') +HASH?=$(shell git ls-tree --full-tree HEAD -- $(CURDIR) | awk '{print $$3}') BASE_DEPS=Dockerfile Makefile -DIRTY=$(shell git diff-index --quiet HEAD -- ../$(notdir $(CURDIR)) || echo "-dirty") +DIRTY=$(shell git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty") TAG=$(ORG)/$(IMAGE):$(HASH)$(DIRTY) # Get a release tag, if present From a315e0b68188066283e4ae67b5f8f781ae3e5947 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Mon, 19 Jun 2017 10:24:31 +0200 Subject: [PATCH 06/12] Fix DIRS find command to work on more system Signed-off-by: Vincent Demeester --- pkg/Makefile | 2 +- tools/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/Makefile b/pkg/Makefile index 81ed81bce..47480140b 100644 --- a/pkg/Makefile +++ b/pkg/Makefile @@ -1,4 +1,4 @@ -DIRS = $(shell find . -type d -depth 1) +DIRS = $(shell find . -maxdepth 1 -mindepth 1 -type d) .PHONY: clean dirs $(DIRS) push: diff --git a/tools/Makefile b/tools/Makefile index 20c61fb08..94894d076 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,4 +1,4 @@ -DIRS = $(shell find . -type d -depth 1) +DIRS = $(shell find . -maxdepth 1 -mindepth 1 -type d) .PHONY: clean dirs $(DIRS) push: $(DIRS) From bdd7e7163b8661d05748e431186185f3373841d3 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 12:43:36 +0100 Subject: [PATCH 07/12] pkg: Allow override of commit to calculate hash for show-tag Useful to answer the question of what the hash was at the point of some older commit. $ make --no-print-directory -C pkg/init/ show-tag linuxkit/init:36c56f0664d49c5a6adc1120d1bf5ba6ac30b389 $ make --no-print-directory -C pkg/init/ show-tag HASH_COMMIT=4699f80ef73141ee87a77d03f93065e541fab382 linuxkit/init:17693d233dd009b2a3a8d23673cb85969e1dce80 Signed-off-by: Ian Campbell --- pkg/package.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/package.mk b/pkg/package.mk index 6f32c4eb1..5ca8c14c1 100644 --- a/pkg/package.mk +++ b/pkg/package.mk @@ -2,10 +2,13 @@ default: push ORG?=linuxkit -HASH?=$(shell git ls-tree --full-tree HEAD -- $(CURDIR) | awk '{print $$3}') +HASH_COMMIT?=HEAD # Setting this is only really useful with the show-tag target +HASH?=$(shell git ls-tree --full-tree $(HASH_COMMIT) -- $(CURDIR) | awk '{print $$3}') BASE_DEPS=Dockerfile Makefile +ifneq ($(HASH_COMMIT),HEAD) # Others can't be dirty by definition DIRTY=$(shell git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty") +endif TAG=$(ORG)/$(IMAGE):$(HASH)$(DIRTY) # Get a release tag, if present From 7fd155b38dee9ab4abaf92edb6ecb2681aebaa4a Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 14:29:01 +0100 Subject: [PATCH 08/12] pkg: Do not mark user supplied HASH as dirty If a user passes a HASH e.g. HASH=dev then assume they know what they are doing and don't need dirty tracking. Signed-off-by: Ian Campbell --- pkg/package.mk | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/package.mk b/pkg/package.mk index 5ca8c14c1..f1592b563 100644 --- a/pkg/package.mk +++ b/pkg/package.mk @@ -2,15 +2,19 @@ default: push ORG?=linuxkit +ifeq ($(HASH),) HASH_COMMIT?=HEAD # Setting this is only really useful with the show-tag target HASH?=$(shell git ls-tree --full-tree $(HASH_COMMIT) -- $(CURDIR) | awk '{print $$3}') -BASE_DEPS=Dockerfile Makefile ifneq ($(HASH_COMMIT),HEAD) # Others can't be dirty by definition DIRTY=$(shell git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty") endif +endif + TAG=$(ORG)/$(IMAGE):$(HASH)$(DIRTY) +BASE_DEPS=Dockerfile Makefile + # Get a release tag, if present RELEASE=$(shell git tag -l --points-at HEAD) From c9b5cbf89b5ed3935f3f855c5f35bfa3db2b6e73 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 14:36:16 +0100 Subject: [PATCH 09/12] kernel: Calculate HASH and DIRTY in the same way as pkg/package.mk The definition of `$(TAG)` differs from pkg/package.mk and is only the HASH+DIRTY since the full tag is defined by the kernel macro and varies for each kernel. Also `show-tag` is `show-tags` here due to the multiple builds. Individual `show-tag_FOO` rules are provided similar to the `build_FOO` rules. Signed-off-by: Ian Campbell --- kernel/Makefile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/kernel/Makefile b/kernel/Makefile index b9ddfcfdb..2c00fc9a1 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -12,25 +12,21 @@ # which is specific to a given kernel. perf packages are tagged the same way # kernel packages. -# Git tree hash of this directory. Override to force build -HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') # Name and Org on Hub ORG?=linuxkit IMAGE:=kernel IMAGE_PERF:=kernel-perf -# Add '-dirty' to hash if the repository is not clean. make does not -# concatenate strings without spaces, so we use the documented trick -# of replacing the space with nothing. -DIRTY=$(shell git diff-index --quiet HEAD --; echo $$?) -ifneq ($(DIRTY),0) -HASH+=-dirty -nullstring := -space := $(nullstring) $(nullstring) -TAG=$(subst $(space),,$(HASH)) -else -TAG=$(HASH) +ifeq ($(HASH),) +HASH_COMMIT?=HEAD # Setting this is only really useful with the show-tag target +HASH?=$(shell git ls-tree --full-tree $(HASH_COMMIT) -- $(CURDIR) | awk '{print $$3}') + +ifneq ($(HASH_COMMIT),HEAD) # Others can't be dirty by definition +DIRTY=$(shell git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty") endif +endif + +TAG=$(HASH)$(DIRTY) .PHONY: check tag push # Targets: @@ -59,14 +55,18 @@ build_$(2)$(3): Dockerfile Makefile $(wildcard patches-$(2)/*) kernel_config-$(2 --no-cache -t $(ORG)/$(IMAGE):$(1)$(3)-$(TAG) . push_$(2)$(3): build_$(2)$(3) - @if [ $(DIRTY) -ne 0 ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi + @if [ x"$(DIRTY)" != x ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi DOCKER_CONTENT_TRUST=1 docker pull $(ORG)/$(IMAGE):$(1)$(3)-$(TAG) || \ (DOCKER_CONTENT_TRUST=1 docker push $(ORG)/$(IMAGE):$(1)$(3)-$(TAG) && \ docker tag $(ORG)/$(IMAGE):$(1)$(3)-$(TAG) $(ORG)/$(IMAGE):$(1)$(3) && \ DOCKER_CONTENT_TRUST=1 docker push $(ORG)/$(IMAGE):$(1)$(3)) +show-tag_$(2)$(3): + @echo $(ORG)/$(IMAGE):$(1)$(3)-$(TAG) + build: build_$(2)$(3) push: push_$(2)$(3) +show-tags: show-tag_$(2)$(3) ifneq ($(2), 4.4.x) build_perf_$(2)$(3): build_$(2)$(3) @@ -76,7 +76,7 @@ build_perf_$(2)$(3): build_$(2)$(3) --no-cache --network=none -t $(ORG)/$(IMAGE_PERF):$(1)$(3)-$(TAG) . push_perf_$(2)$(3): build_perf_$(2)$(3) - @if [ $(DIRTY) -ne 0 ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi + @if [ x"$(DIRTY)" != x ]; then echo "Your repository is not clean. Will not push image"; exit 1; fi DOCKER_CONTENT_TRUST=1 docker pull $(ORG)/$(IMAGE_PERF):$(1)$(3)-$(TAG) || \ (DOCKER_CONTENT_TRUST=1 docker push $(ORG)/$(IMAGE_PERF):$(1)$(3)-$(TAG) && \ docker tag $(ORG)/$(IMAGE_PERF):$(1)$(3)-$(TAG) $(ORG)/$(IMAGE_PERF):$(1)$(3) && \ From 7534d4c926bace8f37c5bedfcc4ad4c800962605 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 14:49:45 +0100 Subject: [PATCH 10/12] tool/go-compile: Use common pkg/package.mk to drive Makefile Signed-off-by: Ian Campbell --- tools/go-compile/Makefile | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/tools/go-compile/Makefile b/tools/go-compile/Makefile index 1f0995e2a..3caba19e1 100644 --- a/tools/go-compile/Makefile +++ b/tools/go-compile/Makefile @@ -1,15 +1,4 @@ -.PHONY: tag push -default: push +include ../../pkg/package.mk -ORG?=linuxkit IMAGE=go-compile -DEPS=Dockerfile Makefile compile.sh - -HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') - -tag: $(DEPS) - docker build --no-cache --network=none -t $(ORG)/$(IMAGE):$(HASH) . - -push: tag - DOCKER_CONTENT_TRUST=1 docker pull $(ORG)/$(IMAGE):$(HASH) || \ - DOCKER_CONTENT_TRUST=1 docker push $(ORG)/$(IMAGE):$(HASH) +DEPS=compile.sh From d5535ea3e49db459ed381286177708c9b8de917b Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 14:50:01 +0100 Subject: [PATCH 11/12] tools/qemu: Use common pkg/package.mk to drive Makefile Signed-off-by: Ian Campbell --- tools/qemu/Makefile | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tools/qemu/Makefile b/tools/qemu/Makefile index 0496d8d0e..b52fa7214 100644 --- a/tools/qemu/Makefile +++ b/tools/qemu/Makefile @@ -1,15 +1,3 @@ -.PHONY: tag push -default: push +include ../../pkg/package.mk -ORG?=linuxkit IMAGE=qemu -DEPS=Dockerfile Makefile - -HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') - -tag: $(DEPS) - DOCKER_CONTENT_TRUST=1 docker build --no-cache --network=none -t $(ORG)/$(IMAGE):$(HASH) . - -push: tag - docker pull $(ORG)/$(IMAGE):$(HASH) || \ - docker push $(ORG)/$(IMAGE):$(HASH) From fe918f8a3157110ebec3c3fac5cc4e6d3d6038b2 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 30 Jun 2017 13:15:32 +0100 Subject: [PATCH 12/12] pkg: Run `git update-index --refresh` before `git diff-index`. Otherwise files which have an updated timestamp but no actual changes are marked as changes because `git diff-index` only uses the `lstat` result and not the actual file contents. Running `git update-index --refresh` updates the cache. Signed-off-by: Ian Campbell --- kernel/Makefile | 2 +- pkg/package.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/Makefile b/kernel/Makefile index 2c00fc9a1..fc4e1a6be 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -22,7 +22,7 @@ HASH_COMMIT?=HEAD # Setting this is only really useful with the show-tag target HASH?=$(shell git ls-tree --full-tree $(HASH_COMMIT) -- $(CURDIR) | awk '{print $$3}') ifneq ($(HASH_COMMIT),HEAD) # Others can't be dirty by definition -DIRTY=$(shell git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty") +DIRTY=$(shell git update-index -q --refresh && git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty") endif endif diff --git a/pkg/package.mk b/pkg/package.mk index f1592b563..350846778 100644 --- a/pkg/package.mk +++ b/pkg/package.mk @@ -7,7 +7,7 @@ HASH_COMMIT?=HEAD # Setting this is only really useful with the show-tag target HASH?=$(shell git ls-tree --full-tree $(HASH_COMMIT) -- $(CURDIR) | awk '{print $$3}') ifneq ($(HASH_COMMIT),HEAD) # Others can't be dirty by definition -DIRTY=$(shell git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty") +DIRTY=$(shell git update-index -q --refresh && git diff-index --quiet HEAD -- $(CURDIR) || echo "-dirty") endif endif