diff --git a/Makefile b/Makefile index f99dbd447..d6d4da8c2 100644 --- a/Makefile +++ b/Makefile @@ -103,3 +103,15 @@ ci-pr: test-cross .PHONY: clean clean: rm -rf bin *.log *-kernel *-cmdline *-state *.img *.iso *.gz *.qcow2 *.vhd *.vmx *.vmdk *.tar *.raw + +update-package-tags: +ifneq ($(LK_RELEASE),) + $(eval tags := $(shell cd pkg; make show-tag | cut -d ':' -f1)) + $(eval image := :$(LK_RELEASE)) +else + $(eval tags := $(shell cd pkg; make show-tag)) + $(eval image := ) +endif + for img in $(tags); do \ + ./scripts/update-component-sha.sh --image $${img}$(image); \ + done diff --git a/docs/alpine-base-update.md b/docs/alpine-base-update.md new file mode 100644 index 000000000..f8289764f --- /dev/null +++ b/docs/alpine-base-update.md @@ -0,0 +1,232 @@ +# Updating Alpine Base + +This document describes the steps to update the `linuxkit/alpine` image. +This image is at the base of all other linuxkit images. +It is built out of the directory `tools/alpine/`. + +While you do not need to update every downstream image _immediately_ when you update +this image, you do need to be aware that changes to this image will affect the +downstream images when it is next adopted. Those downstream images should be updated +as soon as possible after updating `linuxkit/alpine`. + +When you make a linuxkit release, you _must_ update all of the downstream images. +See [releasing.md](./releasing.md) for the release process. + +## Pre-requisites + +Updating `linuxkit/alpine` can be done by any maintainer. Maintainers need to have +access to build machines for all architectures support by LinuxKit. + +## Process + +At a high-level, we are going to do the following: + +1. Preparatory steps +1. Create a new branch +1. Make our desired changes to `tools/alpine` and commit them +1. Build and push out our alpine changes, and commit the `versions` files +1. Update all affected downstream changes and commit them: `tools/`, `test/pkg`, `pkg`, `test/`, `examples/` +1. Push out all affected downstream changes: `tools/`, `test/pkg`, `pkg`, `test/`, `examples/` + +For each of the affected downstream changes, we could update and then push, then move to the next. However, +since the push out can be slow and require retries, we try to make all of the changes first, and then push them out. + +### Preparation + +As a starting point you have to be on the update to date master branch +and be in the root directory of your local git clone. You should also +have the same setup on all build machines used. + +To make the steps below cut-and-pastable, define the following +environment variables: + +```sh +LK_ROOT=$(pwd) +LK_REMOTE=origin # or whatever your personal remote is +LK_BRANCH=alpine-update # or whatever the name of the branch on which you are working is +``` + +Note that if you are cutting a release, the `LK_BRANCH` may have a release-type name, e.g. `rel_v0.4`. + +Make sure that you have the latest version of the `linuxkit` +utility in the path. Alternatively, you may wish to compile the latest version from +master. + +### Create a new branch + +On one of the build machines (preferably the `x86_64` machine), create +the branch: + +```sh +git checkout -b $LK_BRANCH +``` + +### Update `linuxkit/alpine` + +You must perform the arch-specific image builds, pushes and updates on each +architecture first - these can be done in parallel, if you choose. When done, +you then copy the updated `versions.` to one place, commit them, and +push the manifest. + +#### Make alpine changes + +Make any changes in `tools/alpine` that you desire, then commit them. +In the below, change the commit message to something meaningful to the change you are making. + +```sh +cd tools/alpine +# make changes +git commit -s -a -m "Update linuxkit/alpine" +git push origin $LK_BRANCH +``` + +#### Build and Push Alpine Per-Architecture + +On each supported platform, build and update `linuxkit/alpine`, which will update the `versions.` +file.: + +```sh +git fetch +git checkout $LK_BRANCH +cd $LK_ROOT/tools/alpine +make push +``` + +Repeat on each platform. + +#### Commit Changed Versions Files + +When all of the platforms are done, copy the changed `versions.` from each platform to one place, commit and push. +In the below, replace `linuxkit-arch` with each build machine's name: + +```sh +# one of these will not be necessary, as you will likely be executing it on one of these machines +scp linuxkit-s390x:$LK_ROOT/tools/alpine/versions.s390x $LK_ROOT/tools/alpine/versions.s390x +scp linuxkit-aarch64:$LK_ROOT/tools/alpine/versions.aarch64 $LK_ROOT/tools/alpine/versions.aarch64 +scp linuxkit-x86_64:$LK_ROOT/tools/alpine/versions.x86_64 $LK_ROOT/tools/alpine/versions.x86_64 +git commit -a -s -m "tools/alpine: Update to latest" +git push $LK_REMOTE $LK_BRANCH +``` + +#### Update and Push Multi-Arch Index + +Push out the multi-arch index: + +```sh +make push-manifest +``` + +Stash the tag of the alpine base image in an environment variable: + +```sh +LK_ALPINE=$(make show-tag) +``` + +### Update affected downstream packages + +This section describes all of the steps. Below follows a straight copyable list of steps to take, +following which is an explanation of each one. + +```sh +# Update tools packages +cd $LK_ROOT/tools +$LK_ROOT/scripts/update-component-sha.sh --pkg $LK_ROOT/tools/alpine +git checkout grub/Dockerfile +git commit -a -s -m "tools: Update to the latest linuxkit/alpine" + +# Update tools dependencies +cd $LK_ROOT +for img in $(cd tools; make show-tag); do + $LK_ROOT/scripts/update-component-sha.sh --image $img +done +git commit -a -s -m "Update use of tools to latest" + +# Update test packages +cd $LK_ROOT/test/pkg +$LK_ROOT/scripts/update-component-sha.sh --pkg $LK_ROOT/tools/alpine +git commit -a -s -m "tests: Update packages to the latest linuxkit/alpine" + +# Update test packages dependencies +cd $LK_ROOT +for img in $(cd test/pkg; make show-tag); do + $LK_ROOT/scripts/update-component-sha.sh --image $img +done +git commit -a -s -m "Update use of test packages to latest" + +# Update test cases to latest linuxkit/alpine +cd $LK_ROOT/test/cases +$LK_ROOT/scripts/update-component-sha.sh --pkg $LK_ROOT/tools/alpine +git commit -a -s -m "tests: Update tests cases to the latest linuxkit/alpine" + +# Update packages to latest linuxkit/alpine +cd $LK_ROOT/pkg +$LK_ROOT/scripts/update-component-sha.sh --pkg $LK_ROOT/tools/alpine +git commit -a -s -m "pkgs: Update packages to the latest linuxkit/alpine" + +# update package tags - may want to include the release in it if set +cd $LK_ROOT +make update-package-tags +MSG="" +[ -n "$LK_RELEASE" ] && MSG="to $LK_RELEASE" +git commit -a -s -m "Update package tags $MSG" + +git push $LK_REMOTE $LK_BRANCH +``` + +#### Update tools packages + +On your primary build machine, update the other tools packages. + +Note, the `git checkout` reverts the changes made by +`update-component-sha.sh` to files which are accidentally updated. +Important is the `git checkout` of `grub`. This is a bit old and only can be built with specific +older versions of packages like `gcc`, and should not be updated. + +Then we update any dependencies of these tools. + +#### Update test packages + +Next, we update the test packages to the updated alpine base. + +Next, we update the use of test packages to latest. + +Some tests also use `linuxkit/alpine`, so we update them as well. + +### Update packages + +Next, we update the LinuxKit packages. This is really the core of the +release. The other steps above are just there to ensure consistency +across packages. + +#### External Tools + +Most of the packages are build from `linuxkit/alpine` and source code +in the `linuxkit` repository, but some packages wrap external +tools. When updating all packages, and especially during the time of a release, +is a good opportunity to check if there have been updates. Specifically: + +- `pkg/cadvisor`: Check for [new releases](https://github.com/google/cadvisor/releases). +- `pkg/firmware` and `pkg/firmware-all`: Use latest commit from [here](https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git). +- `pkg/node_exporter`: Check for [new releases](https://github.com/prometheus/node_exporter/releases). +- Check [docker hub](https://hub.docker.com/r/library/docker/tags/) for the latest `dind` tags. and update `examples/docker.yml`, `examples/docker-for-mac.yml`, `examples/cadvisor.yml`, and `test/cases/030_security/000_docker-bench/test.yml` if necessary. + +This is at your discretion. + +### Build and push affected downstream packages + +
    Note
: All of the `make push` and `make forcepush` in this section use `linuxkit pkg push`, which will build for all architectures and push +the images out. See [Build Platforms](./packages.md#Build_Platforms). + +```sh +# build and push out the tools packages +cd $LK_ROOT/tools +make forcepush + +# Build and push out test packages +cd $LK_ROOT/test/pkg +make push + +# build and push out the packages +cd $LK_ROOT/pkg +make push +``` diff --git a/docs/releasing.md b/docs/releasing.md index e24f168ae..088b540f1 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -37,220 +37,18 @@ As a starting point you have to be on the update to date master branch and be in the root directory of your local git clone. You should also have the same setup on all build machines used. -To make the release steps below cut-and-pastable, define the following -environment variables: - -```sh -LK_RELEASE=v0.4 -LK_ROOT=$(pwd) -LK_REMOTE=origin -``` - -On one of the build machines (preferably the `x86_64` machine), create -the release branch: - -```sh -git checkout -b rel_$LK_RELEASE -``` - -Also make sure that you have a recent version of the `linuxkit` -utility in the path. Either a previous release or compiled from -master. - - ### Update `linuxkit/alpine` This step is not necessarily required if the alpine base image has recently been updated, but it is good to pick up any recent bug -fixes. Updating the alpine base image is different to other packages. -You must perform the arch-specific image builds, pushes and updates on each -architecture first - these can be done in parallel, if you choose. When done, -you then copy the updated `versions.` to one place, commit them, and -push the manifest. +fixes. Follow the process in [alpine-base-update.md](./alpine-base-update.md) -Details: +There are several important notes to consider when updating alpine base: -#### Build and Push Per-Architecture - -On each supported platform, build and update `linuxkit/alpine`, which will update the `versions.` -file.: - -```sh -git fetch -git checkout rel_$LK_RELEASE -cd $LK_ROOT/tools/alpine -make push -``` - -Repeat on each platform. - -#### Commit Changed Versions Files - -When all of the platforms are done, copy the changed `versions.` from each platform to one please, commit and push. -In the below, replace `linuxkit-arch` with each build machine's name: - -```sh -# one of these will not be necessary, as you will likely be executing it on one of these machines -scp linuxkit-s390x:$LK_ROOT/tools/alpine/versions.s390x $LK_ROOT/tools/alpine/versions.s390x -scp linuxkit-aarch64:$LK_ROOT/tools/alpine/versions.aarch64 $LK_ROOT/tools/alpine/versions.aarch64 -scp linuxkit-x86_64:$LK_ROOT/tools/alpine/versions.x86_64 $LK_ROOT/tools/alpine/versions.x86_64 -git commit -a -s -m "tools/alpine: Update to latest" -git push $LK_REMOTE rel_$LK_RELEASE -``` - -#### Update and Push Multi-Arch Index - -Push out the multi-arch index: - -```sh -make push-manifest -``` - -Stash the tag of the alpine base image in an environment variable: - -```sh -LK_ALPINE=$(make show-tag) -``` - - -### Update tools packages - -On the `x86_64` machine, get the `linuxkit/alpine` tag and update the -other packages: - -```sh -cd $LK_ROOT/tools -../scripts/update-component-sha.sh --image linuxkit/alpine:$LK_ALPINE -git checkout alpine/versions.aarch64 alpine/versions.s390x -git checkout grub/Dockerfile - -git commit -a -s -m "tools: Update to the latest linuxkit/alpine" -git push $LK_REMOTE rel_$LK_RELEASE - -make forcepush -``` - -Note, the `git checkout` reverts the changes made by -`update-component-sha.sh` to files which are accidentally updated and -the `make forcepush` will skip building the alpine base. -Also, `git checkout` of `grub`. This is a bit old and only can be built with specific -older versions of packages like `gcc`, and should not be updated. - -Then, on the other build machines in turn: - -```sh -cd $LK_ROOT/tools -git fetch && git reset --hard $LK_REMOTE/rel_$LK_RELEASE -make forcepush -``` - -Back on the `x86_64` machine: - -```sh -cd $LK_ROOT -for img in $(cd tools; make show-tag); do - ./scripts/update-component-sha.sh --image $img -done - -git commit -a -s -m "Update use of tools to latest" -``` - - -### Update test packages - -Next, we update the test packages to the updated alpine base on the `x86_64` system: - -```sh -cd $LK_ROOT/test/pkg -../../scripts/update-component-sha.sh --image linuxkit/alpine:$LK_ALPINE - -git commit -a -s -m "tests: Update packages to the latest linuxkit/alpine" -git push $LK_REMOTE rel_$LK_RELEASE - -make push -``` - -Then, on the other build machines in turn: - -```sh -cd $LK_ROOT/test/pkg -git fetch && git reset --hard $LK_REMOTE/rel_$LK_RELEASE -make push -``` - -Back on the `x86_64` machine: - -```sh -cd $LK_ROOT -for img in $(cd test/pkg; make show-tag); do - ./scripts/update-component-sha.sh --image $img -done - -git commit -a -s -m "Update use of test packages to latest" -``` - -Some tests also use `linuxkit/alpine`. Update them as well: - -```sh -cd $LK_ROOT/test/cases -../../scripts/update-component-sha.sh --image linuxkit/alpine:$LK_ALPINE - -git commit -a -s -m "tests: Update tests cases to the latest linuxkit/alpine" -``` - -### Update packages - -Next, we update the LinuxKit packages. This is really the core of the -release. The other steps above are just there to ensure consistency -across packages. - - -```sh -cd $LK_ROOT/pkg -../scripts/update-component-sha.sh --image linuxkit/alpine:$LK_ALPINE - -git commit -a -s -m "pkgs: Update packages to the latest linuxkit/alpine" -git push $LK_REMOTE rel_$LK_RELEASE -``` - -Most of the packages are build from `linuxkit/alpine` and source code -in the `linuxkit` repository, but some packages wrap external -tools. The time of a release is a good opportunity to check if there -have been updates. Specifically: - -- `pkg/cadvisor`: Check for [new releases](https://github.com/google/cadvisor/releases). -- `pkg/firmware` and `pkg/firmware-all`: Use latest commit from [here](https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git). -- `pkg/node_exporter`: Check for [new releases](https://github.com/prometheus/node_exporter/releases). -- Check [docker hub](https://hub.docker.com/r/library/docker/tags/) for the latest `dind` tags. and update `examples/docker.yml`, `examples/docker-for-mac.yml`, `examples/cadvisor.yml`, and `test/cases/030_security/000_docker-bench/test.yml` if necessary. - -The build/push the packages: - -```sh -cd $LK_ROOT/pkg -make OPTIONS="-release $LK_RELEASE" push -``` - -Note, the `OPTIONS` argument. This adds the release tag to the -packages. - -Then, on the other build machines in turn: - -```sh -cd $LK_ROOT/pkg -git fetch && git reset --hard $LK_REMOTE/rel_$LK_RELEASE -make OPTIONS="-release $LK_RELEASE" push -``` - -Update the package tags in the YAML files: - -```sh -cd $LK_ROOT -for img in $(cd pkg; make show-tag | cut -d ':' -f1); do - ./scripts/update-component-sha.sh --image $img:$LK_RELEASE -done - -git commit -a -s -m "Update package tags to $LK_RELEASE" -``` +* `LK_BRANCH` is set to `rel_$LK_RELEASE`, when cutting a release, for e.g. `LK_BRANCH=rel_v0.9` +* It not necessarily required to update the alpine base image if it has recently been updated, but it is good to pick up any recent bug +fixes. However, you do need to update the tools, packages and tests. +* Releases are a particularly good time to check for updates in wrapped external dependencies, as highlighted in [alpine-base-update.md#External Tools](./alpine-base-update.md#External_Tools) ### Final preparation steps diff --git a/pkg/Makefile b/pkg/Makefile index 4484fd347..93a59c0c4 100644 --- a/pkg/Makefile +++ b/pkg/Makefile @@ -3,19 +3,25 @@ DIRS = $(dir $(shell find . -maxdepth 2 -mindepth 2 -type f -name build.yml)) OPTIONS ?= +PUSHOPTIONS = + +ifneq ($(LK_RELEASE),) +PUSHOPTIONS += -release $(LK_RELEASE) +endif + push: - @set -e; for d in $(DIRS); do linuxkit pkg push $(OPTIONS) "$$d"; done + @linuxkit pkg push $(OPTIONS) $(PUSHOPTIONS) $(DIRS) forcepush: - @set -e; for d in $(DIRS); do linuxkit pkg push $(OPTIONS) --force "$$d"; done + @linuxkit pkg push $(OPTIONS) $(PUSHOPTIONS) --force $(DIRS) build: - @set -e; for d in $(DIRS); do linuxkit pkg build $(OPTIONS) "$$d"; done + @linuxkit pkg build $(OPTIONS) $(DIRS) forcebuild: - @set -e; for d in $(DIRS); do linuxkit pkg build $(OPTIONS) --force "$$d"; done + @linuxkit pkg build $(OPTIONS) --force $(DIRS) show-tag: - @set -e; for d in $(DIRS); do linuxkit pkg show-tag $(OPTIONS) "$$d"; done + @linuxkit pkg show-tag $(OPTIONS) $(DIRS) clean: