From be37a79440275b9a9f011e2a1c91ae8c14f9445a Mon Sep 17 00:00:00 2001
From: Sven Dowideit <SvenDowideit@home.org.au>
Date: Mon, 14 Nov 2016 15:37:18 +1000
Subject: [PATCH 1/2] the header and extras image code from the rancher/os repo

Signed-off-by: Sven Dowideit <SvenDowideit@home.org.au>
---
 images/10-extras/Dockerfile  |  5 +++++
 images/10-extras/extra.sh    | 19 +++++++++++++++++++
 images/10-headers/Dockerfile |  3 +++
 images/10-headers/headers.sh | 24 ++++++++++++++++++++++++
 scripts/build-images         | 25 +++++++++++++++++++++++++
 5 files changed, 76 insertions(+)
 create mode 100644 images/10-extras/Dockerfile
 create mode 100755 images/10-extras/extra.sh
 create mode 100644 images/10-headers/Dockerfile
 create mode 100755 images/10-headers/headers.sh
 create mode 100755 scripts/build-images

diff --git a/images/10-extras/Dockerfile b/images/10-extras/Dockerfile
new file mode 100644
index 0000000..ceeb57c
--- /dev/null
+++ b/images/10-extras/Dockerfile
@@ -0,0 +1,5 @@
+FROM alpine:3.4
+# FROM arm64=skip arm=skip
+RUN apk --purge --no-cache add kmod bash openssl
+COPY extra.sh  /usr/bin
+CMD ["extra.sh"]
diff --git a/images/10-extras/extra.sh b/images/10-extras/extra.sh
new file mode 100755
index 0000000..eb0b8af
--- /dev/null
+++ b/images/10-extras/extra.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+DIR=$(readlink /lib/modules/$(uname -r)/build)
+STAMP=/lib/modules/$(uname -r)/.extra-done
+VER=$(basename $DIR)
+URL=${KERNEL_EXTRAS_URL:-https://github.com/rancher/os-kernel/releases/download/${VER}/extra.tar.gz}
+
+if [ -e $STAMP ]; then
+    echo Kernel extras already installed. Delete $STAMP to reinstall
+    exit 0
+fi
+
+echo Downloading $URL
+wget -O - $URL | gzip -dc | tar xf - -C /
+depmod -a
+touch $STAMP
+
+echo Kernel extras installed
diff --git a/images/10-headers/Dockerfile b/images/10-headers/Dockerfile
new file mode 100644
index 0000000..be9712a
--- /dev/null
+++ b/images/10-headers/Dockerfile
@@ -0,0 +1,3 @@
+FROM rancher/os-base
+COPY headers.sh  /
+CMD ["/headers.sh"]
diff --git a/images/10-headers/headers.sh b/images/10-headers/headers.sh
new file mode 100755
index 0000000..6f6f84c
--- /dev/null
+++ b/images/10-headers/headers.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+DIR=$(readlink /lib/modules/$(uname -r)/build)
+STAMP=${DIR}/.done
+VER=$(basename $DIR)
+
+if [ "$VER" = "Ubuntu-4.4.0-23.41-rancher2" ]; then
+    VER=Ubuntu-4.4.0-23.41-rancher2-2
+fi
+
+KERNEL_HEADERS_URL=${KERNEL_HEADERS_URL:-https://github.com/rancher/os-kernel/releases/download/${VER}/build.tar.gz}
+
+if [ -e $STAMP ]; then
+    echo Headers already installed in $DIR
+    exit 0
+fi
+
+echo Downloading $KERNEL_HEADERS_URL
+mkdir -p $DIR
+wget -O - $KERNEL_HEADERS_URL | gzip -dc | tar xf - -C $DIR
+touch $STAMP
+
+echo Headers installed at $DIR
diff --git a/scripts/build-images b/scripts/build-images
new file mode 100755
index 0000000..26a87c8
--- /dev/null
+++ b/scripts/build-images
@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+export ARCH=${ARCH:-"amd64"}
+BASE=images
+
+source $(dirname $0)/version
+cd $(dirname $0)/..
+
+for i in $BASE/[0-9]*; do
+    name="os-$(echo ${i} | cut -f2 -d-)"
+    tag="${OS_REPO}/${name}:${VERSION}${SUFFIX}"
+    echo Building ${tag}
+    if [ -x ${i}/prebuild.sh ]; then
+        ${i}/prebuild.sh
+    fi
+
+    if dapper -d --build -f ${i}/Dockerfile -- -t rancher/${name} ${i}; then
+        docker tag rancher/${name} ${tag}
+    elif [ "$?" != "42" ]; then
+        exit 1
+    else
+        echo "WARN: Skipping ${tag}"
+    fi
+done

From 62bdf18b9d2c0a5701cd89b0b68932bb46236a4f Mon Sep 17 00:00:00 2001
From: Sven Dowideit <SvenDowideit@home.org.au>
Date: Mon, 14 Nov 2016 20:10:09 +1000
Subject: [PATCH 2/2] os-extras and os-headers images are built for release/ci

Signed-off-by: Sven Dowideit <SvenDowideit@home.org.au>
---
 .dockerignore                 |  1 -
 Dockerfile.dapper             | 61 +++++++++++++++++++++++++++++++----
 Makefile                      |  9 ++----
 images/10-extras/Dockerfile   |  7 +++-
 images/10-extras/extra.sh     | 20 ++++++------
 images/10-extras/prebuild.sh  |  6 ++++
 images/10-headers/Dockerfile  |  9 +++++-
 images/10-headers/headers.sh  | 21 ++++++------
 images/10-headers/prebuild.sh |  6 ++++
 scripts/build-images          | 15 ++++++---
 scripts/build-kernel          | 18 ++++++++---
 scripts/ci                    | 11 +++++++
 scripts/download              |  1 +
 scripts/package-kernel        | 34 ++++++++++++-------
 14 files changed, 161 insertions(+), 58 deletions(-)
 create mode 100755 images/10-extras/prebuild.sh
 create mode 100755 images/10-headers/prebuild.sh

diff --git a/.dockerignore b/.dockerignore
index 6db8c35..cc3842c 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,5 +1,4 @@
 build
-dist
 .buildroot-ccache
 .kernel-ccache
 .dl
diff --git a/Dockerfile.dapper b/Dockerfile.dapper
index 35ec61a..7e820b9 100644
--- a/Dockerfile.dapper
+++ b/Dockerfile.dapper
@@ -1,20 +1,69 @@
 FROM ubuntu:16.04
+# FROM arm64=aarch64/ubuntu:16.04 arm=armhf/ubuntu:16.04
 
-RUN apt-get update && \
-    apt-get install -y build-essential wget libncurses5-dev unzip bc curl python rsync ccache git vim libssl-dev kmod
+ENV DAPPER_ENV ARCH VERSION DEV_BUILD GITHUB_TOKEN
+ENV DAPPER_DOCKER_SOCKET true
+ENV DAPPER_SOURCE /source
+ENV DAPPER_OUTPUT ./dist
+#ENV DAPPER_RUN_ARGS --privileged
+
+######################
+ARG DAPPER_HOST_ARCH=amd64
+ARG HOST_ARCH=${DAPPER_HOST_ARCH}
+ARG ARCH=${HOST_ARCH}
+
+RUN apt-get update \
+    && apt-get install -y build-essential wget libncurses5-dev unzip bc curl python rsync ccache git vim libssl-dev kmod
+
+# Install dapper
+RUN curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m | sed 's/arm.*/arm/') > /usr/bin/dapper \
+    && chmod +x /usr/bin/dapper
+
+ENTRYPOINT ["./scripts/entry"]
+CMD ["ci"]
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 
-ENV DAPPER_SOURCE /source
-ENV DAPPER_OUTPUT ./dist
 ENV SHELL /bin/bash
 ENV HOME ${DAPPER_SOURCE}
 WORKDIR ${DAPPER_SOURCE}
 
+########## General Configuration #####################
+
+ARG OS_REPO=rancher
+ARG DOCKER_VERSION=1.10.3
+ARG DOCKER_PATCH_VERSION=v${DOCKER_VERSION}-ros1
+
+ARG DOCKER_URL_amd64=https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}
+ARG DOCKER_URL_arm=https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm
+ARG DOCKER_URL_arm64=https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm64
+
+######################################################
+
+# Export all ARGS as ENV
+ENV DOCKER_URL=DOCKER_URL_${ARCH} \
+    ARCH=${ARCH} \
+    DISTRIB_ID=${DISTRIB_ID} \
+    DOCKER_PATCH_VERSION=${DOCKER_PATCH_VERSION} \
+    DOCKER_URL=${DOCKER_URL} \
+    DOCKER_URL_amd64=${DOCKER_URL_amd64} \
+    DOCKER_URL_arm64=${DOCKER_URL_arm64} \
+    DOCKER_URL_arm=${DOCKER_URL_arm} \
+    DOCKER_VERSION=${DOCKER_VERSION} \
+    HOST_ARCH=${HOST_ARCH} \
+    OS_REPO=${OS_REPO}
+
+#RUN rm /bin/sh && \
+#    ln -s /bin/bash /bin/sh
+
+# Install Docker
+RUN curl -fL ${DOCKER_URL_amd64} > /usr/bin/docker && \
+    chmod +x /usr/bin/docker
+
+########## Kernel version Configuration #############################
 ENV KERNEL_TAG=Ubuntu-4.4.0-47.68-rancher1
+ENV KERNEL_VERSION=4.4.24-rancher
 ENV KERNEL_URL=https://github.com/rancher/linux/archive/${KERNEL_TAG}.tar.gz
 ENV KERNEL_SHA1=9e93ea7fc7acf9988572c1961a32ddc17e042b50
 
-ENTRYPOINT ["./scripts/entry"]
-CMD ["ci"]
diff --git a/Makefile b/Makefile
index d7d72a1..641fb1d 100644
--- a/Makefile
+++ b/Makefile
@@ -10,13 +10,8 @@ TARGETS := $(shell ls scripts)
 $(TARGETS): .dapper
 	./.dapper $@
 
-trash: .dapper
-	./.dapper -m bind trash
-
-trash-keep: .dapper
-	./.dapper -m bind trash -k
-
-deps: trash
+shell-bind: .dapper
+	./.dapper -m bind -s
 
 .DEFAULT_GOAL := ci
 
diff --git a/images/10-extras/Dockerfile b/images/10-extras/Dockerfile
index ceeb57c..ea59a70 100644
--- a/images/10-extras/Dockerfile
+++ b/images/10-extras/Dockerfile
@@ -1,5 +1,10 @@
 FROM alpine:3.4
 # FROM arm64=skip arm=skip
-RUN apk --purge --no-cache add kmod bash openssl
+RUN apk --purge --no-cache add kmod bash
+
+ARG KERNEL_VERSION
+ENV KERNEL_VERSION=${KERNEL_VERSION}
+
 COPY extra.sh  /usr/bin
+COPY /extras.tar.gz /
 CMD ["extra.sh"]
diff --git a/images/10-extras/extra.sh b/images/10-extras/extra.sh
index eb0b8af..476891d 100755
--- a/images/10-extras/extra.sh
+++ b/images/10-extras/extra.sh
@@ -1,19 +1,21 @@
 #!/bin/bash
 set -e
+set -x
 
-DIR=$(readlink /lib/modules/$(uname -r)/build)
-STAMP=/lib/modules/$(uname -r)/.extra-done
-VER=$(basename $DIR)
-URL=${KERNEL_EXTRAS_URL:-https://github.com/rancher/os-kernel/releases/download/${VER}/extra.tar.gz}
+echo "Kernel extras for ${KERNEL_VERSION}"
+
+#DIR=/lib/modules/${KERNEL_VERSION}/build
+STAMP=/lib/modules/${KERNEL_VERSION}/.extras-done
 
 if [ -e $STAMP ]; then
-    echo Kernel extras already installed. Delete $STAMP to reinstall
+    echo Kernel extras for ${KERNEL_VERSION} already installed. Delete $STAMP to reinstall
     exit 0
 fi
 
-echo Downloading $URL
-wget -O - $URL | gzip -dc | tar xf - -C /
-depmod -a
+cat /extras.tar.gz | gzip -dc | tar xf - -C /
+if [ "${KERNEL_VERSION}" == "$(uname -r)" ]; then
+    depmod -a
+fi
 touch $STAMP
 
-echo Kernel extras installed
+echo Kernel extras for ${KERNEL_VERSION} installed. Delete $STAMP to reinstall
diff --git a/images/10-extras/prebuild.sh b/images/10-extras/prebuild.sh
new file mode 100755
index 0000000..cfdb95a
--- /dev/null
+++ b/images/10-extras/prebuild.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+set -x
+
+cp /source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz ./images/10-extras/extras.tar.gz
+
diff --git a/images/10-headers/Dockerfile b/images/10-headers/Dockerfile
index be9712a..4f7c9a4 100644
--- a/images/10-headers/Dockerfile
+++ b/images/10-headers/Dockerfile
@@ -1,3 +1,10 @@
-FROM rancher/os-base
+FROM alpine:3.4
+# FROM arm64=skip arm=skip
+RUN apk --purge --no-cache add kmod bash
+
+ARG KERNEL_VERSION
+ENV KERNEL_VERSION=${KERNEL_VERSION}
+
 COPY headers.sh  /
+COPY /build.tar.gz /
 CMD ["/headers.sh"]
diff --git a/images/10-headers/headers.sh b/images/10-headers/headers.sh
index 6f6f84c..53f3919 100755
--- a/images/10-headers/headers.sh
+++ b/images/10-headers/headers.sh
@@ -1,24 +1,21 @@
 #!/bin/bash
 set -e
+set -x
 
-DIR=$(readlink /lib/modules/$(uname -r)/build)
-STAMP=${DIR}/.done
-VER=$(basename $DIR)
+echo "Kernel headers for ${KERNEL_VERSION}"
 
-if [ "$VER" = "Ubuntu-4.4.0-23.41-rancher2" ]; then
-    VER=Ubuntu-4.4.0-23.41-rancher2-2
-fi
-
-KERNEL_HEADERS_URL=${KERNEL_HEADERS_URL:-https://github.com/rancher/os-kernel/releases/download/${VER}/build.tar.gz}
+DIR=/lib/modules/${KERNEL_VERSION}/build
+STAMP=/lib/modules/${KERNEL_VERSION}/.headers-done
 
 if [ -e $STAMP ]; then
-    echo Headers already installed in $DIR
+    echo Kernel headers for ${KERNEL_VERSION} already installed. Delete $STAMP to reinstall
     exit 0
 fi
 
-echo Downloading $KERNEL_HEADERS_URL
+rm -rf $DIR
 mkdir -p $DIR
-wget -O - $KERNEL_HEADERS_URL | gzip -dc | tar xf - -C $DIR
+
+cat /headers.tar.gz | gzip -dc | tar xf - -C $DIR
 touch $STAMP
 
-echo Headers installed at $DIR
+echo Kernel headers for ${KERNEL_VERSION} installed. Delete $STAMP to reinstall
diff --git a/images/10-headers/prebuild.sh b/images/10-headers/prebuild.sh
new file mode 100755
index 0000000..3602989
--- /dev/null
+++ b/images/10-headers/prebuild.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+set -x
+
+cp /source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz ./images/10-headers/build.tar.gz
+
diff --git a/scripts/build-images b/scripts/build-images
index 26a87c8..93d2665 100755
--- a/scripts/build-images
+++ b/scripts/build-images
@@ -7,19 +7,24 @@ BASE=images
 source $(dirname $0)/version
 cd $(dirname $0)/..
 
-for i in $BASE/[0-9]*; do
+mkdir -p dist
+> dist/images
+
+for i in $BASE/[1-9]*; do
     name="os-$(echo ${i} | cut -f2 -d-)"
-    tag="${OS_REPO}/${name}:${VERSION}${SUFFIX}"
+    tag="${OS_REPO}/${name}:${KERNEL_VERSION}${SUFFIX}"
     echo Building ${tag}
-    if [ -x ${i}/prebuild.sh ]; then
+    if [ -e ${i}/prebuild.sh ]; then
+        echo "running ${i}/prebuild.sh in $(pwd)"
         ${i}/prebuild.sh
     fi
 
-    if dapper -d --build -f ${i}/Dockerfile -- -t rancher/${name} ${i}; then
+    if dapper -d --build -f ${i}/Dockerfile -- -t rancher/${name} --build-arg KERNEL_VERSION ${i}; then
         docker tag rancher/${name} ${tag}
+        echo ${tag} >> dist/images
     elif [ "$?" != "42" ]; then
         exit 1
     else
-        echo "WARN: Skipping ${tag}"
+        echo "Skipping ${tag}"
     fi
 done
diff --git a/scripts/build-kernel b/scripts/build-kernel
index 355e29e..df61569 100755
--- a/scripts/build-kernel
+++ b/scripts/build-kernel
@@ -15,7 +15,17 @@ FIRMWARE=$(readlink -f scripts/firmware)
 MODULE_LIST=$(readlink -f modules.list)
 MODULE_EXTRA_LIST=$(readlink -f modules-extra.list)
 
-cd ${BUILD}/${DIR}
-
-make oldconfig
-make -j$(nproc) tar-pkg
+if [ -e "/source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz" ]; then
+	echo "skipping kernel build - found linux-${KERNEL_VERSION}-rancher-x86.tar.gz in ./dist/kernel/"
+else
+	cd ${BUILD}/${DIR}
+	make oldconfig
+	KVER=$(make kernelrelease)
+	if [ -e "linux-${KERNEL_VERSION}-x86.tar" ] \
+	   && [ "$KVER" == "$KERNEL_VERSION" ]; then
+		echo "Skipping kernel build, found linux-${KERNEL_VERSION}-x86.tar in $(pwd)"
+	else
+		echo "building $KVER"
+		make -j$(nproc) tar-pkg
+	fi
+fi
diff --git a/scripts/ci b/scripts/ci
index 4be07e0..065d824 100755
--- a/scripts/ci
+++ b/scripts/ci
@@ -7,3 +7,14 @@ cd $(dirname $0)/..
 ./scripts/extract
 ./scripts/build-kernel
 ./scripts/package-kernel
+./scripts/build-images
+
+echo
+echo "--- ${KERNEL_VERSION} Kernel prepared for RancherOS"
+echo "	./dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz"
+echo "	./dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz"
+echo "	./dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz"
+echo
+echo "Images ready to push:"
+cat dist/images
+echo
diff --git a/scripts/download b/scripts/download
index 494f4ef..0b1c0ce 100755
--- a/scripts/download
+++ b/scripts/download
@@ -38,4 +38,5 @@ download()
     fi
 }
 
+# Download Kernel source
 download ${KERNEL_SHA1} ${KERNEL_URL}
diff --git a/scripts/package-kernel b/scripts/package-kernel
index 827c42a..54ac87f 100755
--- a/scripts/package-kernel
+++ b/scripts/package-kernel
@@ -30,20 +30,30 @@ FIRMWARE=$(readlink -f scripts/firmware)
 MODULE_LIST=$(readlink -f modules.list)
 MODULE_EXTRA_LIST=$(readlink -f modules-extra.list)
 
-cd ${SOURCEDIR}
+echo "looking for /source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz"
+echo "looking for /source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz"
+echo "looking for /source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz"
+if [ -e "/source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz" ] \
+   && [ -e "/source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz" ] \
+   && [ -e "/source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz" ]; then
+    echo "Skipping packaging of kernel, extras and headers - already found in ./dist/kernel/"
+else
+    cd ${SOURCEDIR}
+    KVER=$(make kernelrelease)
 
-create_firmware_tar $FIRMWARE
+    create_firmware_tar $FIRMWARE
 
-mkdir -p ${DIST}/kernel
+    mkdir -p ${DIST}/kernel
 
-FILE=$(echo linux*.tar)
+    FILE="linux-${KVER}-x86.tar"
 
-(
-    split_tar $FILE $MODULE_LIST $MODULE_EXTRA_LIST
-)
+    (
+        split_tar $FILE $MODULE_LIST $MODULE_EXTRA_LIST
+    )
 
-tar --concatenate --file=base.tar firmware.tar
-tar --concatenate --file=extra.tar firmware-extra.tar
-cat base.tar | gzip -c > ${DIST}/kernel/${FILE}.gz
-cat extra.tar | gzip -c > ${DIST}/kernel/extra.tar.gz
-list_build_files | tar -czf ${DIST}/kernel/build.tar.gz -T /dev/stdin
+    tar --concatenate --file=base.tar firmware.tar
+    tar --concatenate --file=extra.tar firmware-extra.tar
+    cat base.tar | gzip -c > ${DIST}/kernel/${FILE}.gz
+    cat extra.tar | gzip -c > ${DIST}/kernel/extra-${FILE}.gz
+    list_build_files | tar -czf ${DIST}/kernel/build-${FILE}.gz -T /dev/stdin
+fi