os-extras and os-headers images are built for release/ci

Signed-off-by: Sven Dowideit <SvenDowideit@home.org.au>

.dockerignore

@@ -1,5 +1,4 @@
 build
-dist
 .buildroot-ccache
 .kernel-ccache
 .dl

Dockerfile.dapper

@@ -1,20 +1,69 @@
 FROM ubuntu:16.04
+# FROM arm64=aarch64/ubuntu:16.04 arm=armhf/ubuntu:16.04
 
-RUN apt-get update && \
+ENV DAPPER_ENV ARCH VERSION DEV_BUILD GITHUB_TOKEN
-    apt-get install -y build-essential wget libncurses5-dev unzip bc curl python rsync ccache git vim libssl-dev kmod
+ENV DAPPER_DOCKER_SOCKET true
+ENV DAPPER_SOURCE /source
+ENV DAPPER_OUTPUT ./dist
+#ENV DAPPER_RUN_ARGS --privileged
+
+######################
+ARG DAPPER_HOST_ARCH=amd64
+ARG HOST_ARCH=${DAPPER_HOST_ARCH}
+ARG ARCH=${HOST_ARCH}
+
+RUN apt-get update \
+    && apt-get install -y build-essential wget libncurses5-dev unzip bc curl python rsync ccache git vim libssl-dev kmod
+
+# Install dapper
+RUN curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m | sed 's/arm.*/arm/') > /usr/bin/dapper \
+    && chmod +x /usr/bin/dapper
+
+ENTRYPOINT ["./scripts/entry"]
+CMD ["ci"]
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 
-ENV DAPPER_SOURCE /source
-ENV DAPPER_OUTPUT ./dist
 ENV SHELL /bin/bash
 ENV HOME ${DAPPER_SOURCE}
 WORKDIR ${DAPPER_SOURCE}
 
+########## General Configuration #####################
+
+ARG OS_REPO=rancher
+ARG DOCKER_VERSION=1.10.3
+ARG DOCKER_PATCH_VERSION=v${DOCKER_VERSION}-ros1
+
+ARG DOCKER_URL_amd64=https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}
+ARG DOCKER_URL_arm=https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm
+ARG DOCKER_URL_arm64=https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm64
+
+######################################################
+
+# Export all ARGS as ENV
+ENV DOCKER_URL=DOCKER_URL_${ARCH} \
+    ARCH=${ARCH} \
+    DISTRIB_ID=${DISTRIB_ID} \
+    DOCKER_PATCH_VERSION=${DOCKER_PATCH_VERSION} \
+    DOCKER_URL=${DOCKER_URL} \
+    DOCKER_URL_amd64=${DOCKER_URL_amd64} \
+    DOCKER_URL_arm64=${DOCKER_URL_arm64} \
+    DOCKER_URL_arm=${DOCKER_URL_arm} \
+    DOCKER_VERSION=${DOCKER_VERSION} \
+    HOST_ARCH=${HOST_ARCH} \
+    OS_REPO=${OS_REPO}
+
+#RUN rm /bin/sh && \
+#    ln -s /bin/bash /bin/sh
+
+# Install Docker
+RUN curl -fL ${DOCKER_URL_amd64} > /usr/bin/docker && \
+    chmod +x /usr/bin/docker
+
+########## Kernel version Configuration #############################
 ENV KERNEL_TAG=Ubuntu-4.4.0-47.68-rancher1
+ENV KERNEL_VERSION=4.4.24-rancher
 ENV KERNEL_URL=https://github.com/rancher/linux/archive/${KERNEL_TAG}.tar.gz
 ENV KERNEL_SHA1=9e93ea7fc7acf9988572c1961a32ddc17e042b50
 
-ENTRYPOINT ["./scripts/entry"]
-CMD ["ci"]

Makefile

@@ -10,13 +10,8 @@ TARGETS := $(shell ls scripts)
 $(TARGETS): .dapper
 	./.dapper $@
 
-trash: .dapper
+shell-bind: .dapper
-	./.dapper -m bind trash
+	./.dapper -m bind -s
-
-trash-keep: .dapper
-	./.dapper -m bind trash -k
-
-deps: trash
 
 .DEFAULT_GOAL := ci
 

images/10-extras/Dockerfile

@@ -1,5 +1,10 @@
 FROM alpine:3.4
 # FROM arm64=skip arm=skip
-RUN apk --purge --no-cache add kmod bash openssl
+RUN apk --purge --no-cache add kmod bash
+
+ARG KERNEL_VERSION
+ENV KERNEL_VERSION=${KERNEL_VERSION}
+
 COPY extra.sh  /usr/bin
+COPY /extras.tar.gz /
 CMD ["extra.sh"]

images/10-extras/extra.sh

@@ -1,19 +1,21 @@
 #!/bin/bash
 set -e
+set -x
 
-DIR=$(readlink /lib/modules/$(uname -r)/build)
+echo "Kernel extras for ${KERNEL_VERSION}"
-STAMP=/lib/modules/$(uname -r)/.extra-done
+
-VER=$(basename $DIR)
+#DIR=/lib/modules/${KERNEL_VERSION}/build
-URL=${KERNEL_EXTRAS_URL:-https://github.com/rancher/os-kernel/releases/download/${VER}/extra.tar.gz}
+STAMP=/lib/modules/${KERNEL_VERSION}/.extras-done
 
 if [ -e $STAMP ]; then
-    echo Kernel extras already installed. Delete $STAMP to reinstall
+    echo Kernel extras for ${KERNEL_VERSION} already installed. Delete $STAMP to reinstall
     exit 0
 fi
 
-echo Downloading $URL
+cat /extras.tar.gz | gzip -dc | tar xf - -C /
-wget -O - $URL | gzip -dc | tar xf - -C /
+if [ "${KERNEL_VERSION}" == "$(uname -r)" ]; then
-depmod -a
+    depmod -a
+fi
 touch $STAMP
 
-echo Kernel extras installed
+echo Kernel extras for ${KERNEL_VERSION} installed. Delete $STAMP to reinstall

images/10-extras/prebuild.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+set -x
+
+cp /source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz ./images/10-extras/extras.tar.gz
+

images/10-headers/Dockerfile

@@ -1,3 +1,10 @@
-FROM rancher/os-base
+FROM alpine:3.4
+# FROM arm64=skip arm=skip
+RUN apk --purge --no-cache add kmod bash
+
+ARG KERNEL_VERSION
+ENV KERNEL_VERSION=${KERNEL_VERSION}
+
 COPY headers.sh  /
+COPY /build.tar.gz /
 CMD ["/headers.sh"]

images/10-headers/headers.sh

@@ -1,24 +1,21 @@
 #!/bin/bash
 set -e
+set -x
 
-DIR=$(readlink /lib/modules/$(uname -r)/build)
+echo "Kernel headers for ${KERNEL_VERSION}"
-STAMP=${DIR}/.done
-VER=$(basename $DIR)
 
-if [ "$VER" = "Ubuntu-4.4.0-23.41-rancher2" ]; then
+DIR=/lib/modules/${KERNEL_VERSION}/build
-    VER=Ubuntu-4.4.0-23.41-rancher2-2
+STAMP=/lib/modules/${KERNEL_VERSION}/.headers-done
-fi
-
-KERNEL_HEADERS_URL=${KERNEL_HEADERS_URL:-https://github.com/rancher/os-kernel/releases/download/${VER}/build.tar.gz}
 
 if [ -e $STAMP ]; then
-    echo Headers already installed in $DIR
+    echo Kernel headers for ${KERNEL_VERSION} already installed. Delete $STAMP to reinstall
     exit 0
 fi
 
-echo Downloading $KERNEL_HEADERS_URL
+rm -rf $DIR
 mkdir -p $DIR
-wget -O - $KERNEL_HEADERS_URL | gzip -dc | tar xf - -C $DIR
+
+cat /headers.tar.gz | gzip -dc | tar xf - -C $DIR
 touch $STAMP
 
-echo Headers installed at $DIR
+echo Kernel headers for ${KERNEL_VERSION} installed. Delete $STAMP to reinstall

images/10-headers/prebuild.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+set -x
+
+cp /source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz ./images/10-headers/build.tar.gz
+

scripts/build-images

@@ -7,19 +7,24 @@ BASE=images
 source $(dirname $0)/version
 cd $(dirname $0)/..
 
-for i in $BASE/[0-9]*; do
+mkdir -p dist
+> dist/images
+
+for i in $BASE/[1-9]*; do
     name="os-$(echo ${i} | cut -f2 -d-)"
-    tag="${OS_REPO}/${name}:${VERSION}${SUFFIX}"
+    tag="${OS_REPO}/${name}:${KERNEL_VERSION}${SUFFIX}"
     echo Building ${tag}
-    if [ -x ${i}/prebuild.sh ]; then
+    if [ -e ${i}/prebuild.sh ]; then
+        echo "running ${i}/prebuild.sh in $(pwd)"
         ${i}/prebuild.sh
     fi
 
-    if dapper -d --build -f ${i}/Dockerfile -- -t rancher/${name} ${i}; then
+    if dapper -d --build -f ${i}/Dockerfile -- -t rancher/${name} --build-arg KERNEL_VERSION ${i}; then
         docker tag rancher/${name} ${tag}
+        echo ${tag} >> dist/images
     elif [ "$?" != "42" ]; then
         exit 1
     else
-        echo "WARN: Skipping ${tag}"
+        echo "Skipping ${tag}"
     fi
 done

scripts/build-kernel

@@ -15,7 +15,17 @@ FIRMWARE=$(readlink -f scripts/firmware)
 MODULE_LIST=$(readlink -f modules.list)
 MODULE_EXTRA_LIST=$(readlink -f modules-extra.list)
 
-cd ${BUILD}/${DIR}
+if [ -e "/source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz" ]; then
-
+	echo "skipping kernel build - found linux-${KERNEL_VERSION}-rancher-x86.tar.gz in ./dist/kernel/"
-make oldconfig
+else
-make -j$(nproc) tar-pkg
+	cd ${BUILD}/${DIR}
+	make oldconfig
+	KVER=$(make kernelrelease)
+	if [ -e "linux-${KERNEL_VERSION}-x86.tar" ] \
+	   && [ "$KVER" == "$KERNEL_VERSION" ]; then
+		echo "Skipping kernel build, found linux-${KERNEL_VERSION}-x86.tar in $(pwd)"
+	else
+		echo "building $KVER"
+		make -j$(nproc) tar-pkg
+	fi
+fi

scripts/ci

@@ -7,3 +7,14 @@ cd $(dirname $0)/..
 ./scripts/extract
 ./scripts/build-kernel
 ./scripts/package-kernel
+./scripts/build-images
+
+echo
+echo "--- ${KERNEL_VERSION} Kernel prepared for RancherOS"
+echo "	./dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz"
+echo "	./dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz"
+echo "	./dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz"
+echo
+echo "Images ready to push:"
+cat dist/images
+echo

scripts/download

@@ -38,4 +38,5 @@ download()
     fi
 }
 
+# Download Kernel source
 download ${KERNEL_SHA1} ${KERNEL_URL}

scripts/package-kernel

@@ -30,20 +30,30 @@ FIRMWARE=$(readlink -f scripts/firmware)
 MODULE_LIST=$(readlink -f modules.list)
 MODULE_EXTRA_LIST=$(readlink -f modules-extra.list)
 
-cd ${SOURCEDIR}
+echo "looking for /source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz"
+echo "looking for /source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz"
+echo "looking for /source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz"
+if [ -e "/source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz" ] \
+   && [ -e "/source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz" ] \
+   && [ -e "/source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz" ]; then
+    echo "Skipping packaging of kernel, extras and headers - already found in ./dist/kernel/"
+else
+    cd ${SOURCEDIR}
+    KVER=$(make kernelrelease)
 
-create_firmware_tar $FIRMWARE
+    create_firmware_tar $FIRMWARE
 
-mkdir -p ${DIST}/kernel
+    mkdir -p ${DIST}/kernel
 
-FILE=$(echo linux*.tar)
+    FILE="linux-${KVER}-x86.tar"
 
-(
+    (
         split_tar $FILE $MODULE_LIST $MODULE_EXTRA_LIST
-)
+    )
 
-tar --concatenate --file=base.tar firmware.tar
+    tar --concatenate --file=base.tar firmware.tar
-tar --concatenate --file=extra.tar firmware-extra.tar
+    tar --concatenate --file=extra.tar firmware-extra.tar
-cat base.tar | gzip -c > ${DIST}/kernel/${FILE}.gz
+    cat base.tar | gzip -c > ${DIST}/kernel/${FILE}.gz
-cat extra.tar | gzip -c > ${DIST}/kernel/extra.tar.gz
+    cat extra.tar | gzip -c > ${DIST}/kernel/extra-${FILE}.gz
-list_build_files | tar -czf ${DIST}/kernel/build.tar.gz -T /dev/stdin
+    list_build_files | tar -czf ${DIST}/kernel/build-${FILE}.gz -T /dev/stdin
+fi