diff --git a/.dockerignore b/.dockerignore index 6db8c35..cc3842c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ build -dist .buildroot-ccache .kernel-ccache .dl diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 35ec61a..7e820b9 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -1,20 +1,69 @@ FROM ubuntu:16.04 +# FROM arm64=aarch64/ubuntu:16.04 arm=armhf/ubuntu:16.04 -RUN apt-get update && \ - apt-get install -y build-essential wget libncurses5-dev unzip bc curl python rsync ccache git vim libssl-dev kmod +ENV DAPPER_ENV ARCH VERSION DEV_BUILD GITHUB_TOKEN +ENV DAPPER_DOCKER_SOCKET true +ENV DAPPER_SOURCE /source +ENV DAPPER_OUTPUT ./dist +#ENV DAPPER_RUN_ARGS --privileged + +###################### +ARG DAPPER_HOST_ARCH=amd64 +ARG HOST_ARCH=${DAPPER_HOST_ARCH} +ARG ARCH=${HOST_ARCH} + +RUN apt-get update \ + && apt-get install -y build-essential wget libncurses5-dev unzip bc curl python rsync ccache git vim libssl-dev kmod + +# Install dapper +RUN curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m | sed 's/arm.*/arm/') > /usr/bin/dapper \ + && chmod +x /usr/bin/dapper + +ENTRYPOINT ["./scripts/entry"] +CMD ["ci"] RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 -ENV DAPPER_SOURCE /source -ENV DAPPER_OUTPUT ./dist ENV SHELL /bin/bash ENV HOME ${DAPPER_SOURCE} WORKDIR ${DAPPER_SOURCE} +########## General Configuration ##################### + +ARG OS_REPO=rancher +ARG DOCKER_VERSION=1.10.3 +ARG DOCKER_PATCH_VERSION=v${DOCKER_VERSION}-ros1 + +ARG DOCKER_URL_amd64=https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION} +ARG DOCKER_URL_arm=https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm +ARG DOCKER_URL_arm64=https://github.com/rancher/docker/releases/download/${DOCKER_PATCH_VERSION}/docker-${DOCKER_VERSION}_arm64 + +###################################################### + +# Export all ARGS as ENV +ENV DOCKER_URL=DOCKER_URL_${ARCH} \ + ARCH=${ARCH} \ + DISTRIB_ID=${DISTRIB_ID} \ + DOCKER_PATCH_VERSION=${DOCKER_PATCH_VERSION} \ + DOCKER_URL=${DOCKER_URL} \ + DOCKER_URL_amd64=${DOCKER_URL_amd64} \ + DOCKER_URL_arm64=${DOCKER_URL_arm64} \ + DOCKER_URL_arm=${DOCKER_URL_arm} \ + DOCKER_VERSION=${DOCKER_VERSION} \ + HOST_ARCH=${HOST_ARCH} \ + OS_REPO=${OS_REPO} + +#RUN rm /bin/sh && \ +# ln -s /bin/bash /bin/sh + +# Install Docker +RUN curl -fL ${DOCKER_URL_amd64} > /usr/bin/docker && \ + chmod +x /usr/bin/docker + +########## Kernel version Configuration ############################# ENV KERNEL_TAG=Ubuntu-4.4.0-47.68-rancher1 +ENV KERNEL_VERSION=4.4.24-rancher ENV KERNEL_URL=https://github.com/rancher/linux/archive/${KERNEL_TAG}.tar.gz ENV KERNEL_SHA1=9e93ea7fc7acf9988572c1961a32ddc17e042b50 -ENTRYPOINT ["./scripts/entry"] -CMD ["ci"] diff --git a/Makefile b/Makefile index d7d72a1..641fb1d 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,8 @@ TARGETS := $(shell ls scripts) $(TARGETS): .dapper ./.dapper $@ -trash: .dapper - ./.dapper -m bind trash - -trash-keep: .dapper - ./.dapper -m bind trash -k - -deps: trash +shell-bind: .dapper + ./.dapper -m bind -s .DEFAULT_GOAL := ci diff --git a/images/10-extras/Dockerfile b/images/10-extras/Dockerfile index ceeb57c..ea59a70 100644 --- a/images/10-extras/Dockerfile +++ b/images/10-extras/Dockerfile @@ -1,5 +1,10 @@ FROM alpine:3.4 # FROM arm64=skip arm=skip -RUN apk --purge --no-cache add kmod bash openssl +RUN apk --purge --no-cache add kmod bash + +ARG KERNEL_VERSION +ENV KERNEL_VERSION=${KERNEL_VERSION} + COPY extra.sh /usr/bin +COPY /extras.tar.gz / CMD ["extra.sh"] diff --git a/images/10-extras/extra.sh b/images/10-extras/extra.sh index eb0b8af..476891d 100755 --- a/images/10-extras/extra.sh +++ b/images/10-extras/extra.sh @@ -1,19 +1,21 @@ #!/bin/bash set -e +set -x -DIR=$(readlink /lib/modules/$(uname -r)/build) -STAMP=/lib/modules/$(uname -r)/.extra-done -VER=$(basename $DIR) -URL=${KERNEL_EXTRAS_URL:-https://github.com/rancher/os-kernel/releases/download/${VER}/extra.tar.gz} +echo "Kernel extras for ${KERNEL_VERSION}" + +#DIR=/lib/modules/${KERNEL_VERSION}/build +STAMP=/lib/modules/${KERNEL_VERSION}/.extras-done if [ -e $STAMP ]; then - echo Kernel extras already installed. Delete $STAMP to reinstall + echo Kernel extras for ${KERNEL_VERSION} already installed. Delete $STAMP to reinstall exit 0 fi -echo Downloading $URL -wget -O - $URL | gzip -dc | tar xf - -C / -depmod -a +cat /extras.tar.gz | gzip -dc | tar xf - -C / +if [ "${KERNEL_VERSION}" == "$(uname -r)" ]; then + depmod -a +fi touch $STAMP -echo Kernel extras installed +echo Kernel extras for ${KERNEL_VERSION} installed. Delete $STAMP to reinstall diff --git a/images/10-extras/prebuild.sh b/images/10-extras/prebuild.sh new file mode 100755 index 0000000..cfdb95a --- /dev/null +++ b/images/10-extras/prebuild.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e +set -x + +cp /source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz ./images/10-extras/extras.tar.gz + diff --git a/images/10-headers/Dockerfile b/images/10-headers/Dockerfile index be9712a..4f7c9a4 100644 --- a/images/10-headers/Dockerfile +++ b/images/10-headers/Dockerfile @@ -1,3 +1,10 @@ -FROM rancher/os-base +FROM alpine:3.4 +# FROM arm64=skip arm=skip +RUN apk --purge --no-cache add kmod bash + +ARG KERNEL_VERSION +ENV KERNEL_VERSION=${KERNEL_VERSION} + COPY headers.sh / +COPY /build.tar.gz / CMD ["/headers.sh"] diff --git a/images/10-headers/headers.sh b/images/10-headers/headers.sh index 6f6f84c..53f3919 100755 --- a/images/10-headers/headers.sh +++ b/images/10-headers/headers.sh @@ -1,24 +1,21 @@ #!/bin/bash set -e +set -x -DIR=$(readlink /lib/modules/$(uname -r)/build) -STAMP=${DIR}/.done -VER=$(basename $DIR) +echo "Kernel headers for ${KERNEL_VERSION}" -if [ "$VER" = "Ubuntu-4.4.0-23.41-rancher2" ]; then - VER=Ubuntu-4.4.0-23.41-rancher2-2 -fi - -KERNEL_HEADERS_URL=${KERNEL_HEADERS_URL:-https://github.com/rancher/os-kernel/releases/download/${VER}/build.tar.gz} +DIR=/lib/modules/${KERNEL_VERSION}/build +STAMP=/lib/modules/${KERNEL_VERSION}/.headers-done if [ -e $STAMP ]; then - echo Headers already installed in $DIR + echo Kernel headers for ${KERNEL_VERSION} already installed. Delete $STAMP to reinstall exit 0 fi -echo Downloading $KERNEL_HEADERS_URL +rm -rf $DIR mkdir -p $DIR -wget -O - $KERNEL_HEADERS_URL | gzip -dc | tar xf - -C $DIR + +cat /headers.tar.gz | gzip -dc | tar xf - -C $DIR touch $STAMP -echo Headers installed at $DIR +echo Kernel headers for ${KERNEL_VERSION} installed. Delete $STAMP to reinstall diff --git a/images/10-headers/prebuild.sh b/images/10-headers/prebuild.sh new file mode 100755 index 0000000..3602989 --- /dev/null +++ b/images/10-headers/prebuild.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e +set -x + +cp /source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz ./images/10-headers/build.tar.gz + diff --git a/scripts/build-images b/scripts/build-images index 26a87c8..93d2665 100755 --- a/scripts/build-images +++ b/scripts/build-images @@ -7,19 +7,24 @@ BASE=images source $(dirname $0)/version cd $(dirname $0)/.. -for i in $BASE/[0-9]*; do +mkdir -p dist +> dist/images + +for i in $BASE/[1-9]*; do name="os-$(echo ${i} | cut -f2 -d-)" - tag="${OS_REPO}/${name}:${VERSION}${SUFFIX}" + tag="${OS_REPO}/${name}:${KERNEL_VERSION}${SUFFIX}" echo Building ${tag} - if [ -x ${i}/prebuild.sh ]; then + if [ -e ${i}/prebuild.sh ]; then + echo "running ${i}/prebuild.sh in $(pwd)" ${i}/prebuild.sh fi - if dapper -d --build -f ${i}/Dockerfile -- -t rancher/${name} ${i}; then + if dapper -d --build -f ${i}/Dockerfile -- -t rancher/${name} --build-arg KERNEL_VERSION ${i}; then docker tag rancher/${name} ${tag} + echo ${tag} >> dist/images elif [ "$?" != "42" ]; then exit 1 else - echo "WARN: Skipping ${tag}" + echo "Skipping ${tag}" fi done diff --git a/scripts/build-kernel b/scripts/build-kernel index 355e29e..df61569 100755 --- a/scripts/build-kernel +++ b/scripts/build-kernel @@ -15,7 +15,17 @@ FIRMWARE=$(readlink -f scripts/firmware) MODULE_LIST=$(readlink -f modules.list) MODULE_EXTRA_LIST=$(readlink -f modules-extra.list) -cd ${BUILD}/${DIR} - -make oldconfig -make -j$(nproc) tar-pkg +if [ -e "/source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz" ]; then + echo "skipping kernel build - found linux-${KERNEL_VERSION}-rancher-x86.tar.gz in ./dist/kernel/" +else + cd ${BUILD}/${DIR} + make oldconfig + KVER=$(make kernelrelease) + if [ -e "linux-${KERNEL_VERSION}-x86.tar" ] \ + && [ "$KVER" == "$KERNEL_VERSION" ]; then + echo "Skipping kernel build, found linux-${KERNEL_VERSION}-x86.tar in $(pwd)" + else + echo "building $KVER" + make -j$(nproc) tar-pkg + fi +fi diff --git a/scripts/ci b/scripts/ci index 4be07e0..065d824 100755 --- a/scripts/ci +++ b/scripts/ci @@ -7,3 +7,14 @@ cd $(dirname $0)/.. ./scripts/extract ./scripts/build-kernel ./scripts/package-kernel +./scripts/build-images + +echo +echo "--- ${KERNEL_VERSION} Kernel prepared for RancherOS" +echo " ./dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz" +echo " ./dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz" +echo " ./dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz" +echo +echo "Images ready to push:" +cat dist/images +echo diff --git a/scripts/download b/scripts/download index 494f4ef..0b1c0ce 100755 --- a/scripts/download +++ b/scripts/download @@ -38,4 +38,5 @@ download() fi } +# Download Kernel source download ${KERNEL_SHA1} ${KERNEL_URL} diff --git a/scripts/package-kernel b/scripts/package-kernel index 827c42a..54ac87f 100755 --- a/scripts/package-kernel +++ b/scripts/package-kernel @@ -30,20 +30,30 @@ FIRMWARE=$(readlink -f scripts/firmware) MODULE_LIST=$(readlink -f modules.list) MODULE_EXTRA_LIST=$(readlink -f modules-extra.list) -cd ${SOURCEDIR} +echo "looking for /source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz" +echo "looking for /source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz" +echo "looking for /source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz" +if [ -e "/source/dist/kernel/linux-${KERNEL_VERSION}-x86.tar.gz" ] \ + && [ -e "/source/dist/kernel/extra-linux-${KERNEL_VERSION}-x86.tar.gz" ] \ + && [ -e "/source/dist/kernel/build-linux-${KERNEL_VERSION}-x86.tar.gz" ]; then + echo "Skipping packaging of kernel, extras and headers - already found in ./dist/kernel/" +else + cd ${SOURCEDIR} + KVER=$(make kernelrelease) -create_firmware_tar $FIRMWARE + create_firmware_tar $FIRMWARE -mkdir -p ${DIST}/kernel + mkdir -p ${DIST}/kernel -FILE=$(echo linux*.tar) + FILE="linux-${KVER}-x86.tar" -( - split_tar $FILE $MODULE_LIST $MODULE_EXTRA_LIST -) + ( + split_tar $FILE $MODULE_LIST $MODULE_EXTRA_LIST + ) -tar --concatenate --file=base.tar firmware.tar -tar --concatenate --file=extra.tar firmware-extra.tar -cat base.tar | gzip -c > ${DIST}/kernel/${FILE}.gz -cat extra.tar | gzip -c > ${DIST}/kernel/extra.tar.gz -list_build_files | tar -czf ${DIST}/kernel/build.tar.gz -T /dev/stdin + tar --concatenate --file=base.tar firmware.tar + tar --concatenate --file=extra.tar firmware-extra.tar + cat base.tar | gzip -c > ${DIST}/kernel/${FILE}.gz + cat extra.tar | gzip -c > ${DIST}/kernel/extra-${FILE}.gz + list_build_files | tar -czf ${DIST}/kernel/build-${FILE}.gz -T /dev/stdin +fi