diff --git a/.dockerignore b/.dockerignore index 02b75e5a..b48c7fd1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ bin state build images/*/build +scripts/images/*/dist/ dist tests/integration/.venv* tests/integration/.tox diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 00f45603..0b75f87b 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -28,6 +28,7 @@ RUN apt-get update && \ wget \ xorriso +ENV DAPPER_ENV VERSION DEV_BUILD ENV DAPPER_DOCKER_SOCKET true ENV DAPPER_SOURCE /go/src/github.com/rancher/os ENV DAPPER_OUTPUT ./bin ./dist ./build/initrd @@ -99,10 +100,12 @@ RUN if [ -n "${!VBOX_MODULES_URL}" ]; then \ ;fi # Install Go +ENV GO_VERSION 1.6.2 RUN ln -sf go-6 /usr/bin/go && \ - curl -sfL https://storage.googleapis.com/golang/go1.6.src.tar.gz | tar -xzf - -C /usr/local && \ + curl -sfL https://storage.googleapis.com/golang/go${GO_VERSION}.src.tar.gz | tar -xzf - -C /usr/local && \ cd /usr/local/go/src && \ - GOROOT_BOOTSTRAP=/usr GOARCH=${HOST_ARCH} GOHOSTARCH=${HOST_ARCH} ./make.bash + GOROOT_BOOTSTRAP=/usr GOARCH=${HOST_ARCH} GOHOSTARCH=${HOST_ARCH} ./make.bash && \ + rm /usr/bin/go ENV GOPATH /go ENV PATH ${GOPATH}/bin:/usr/local/go/bin:$PATH diff --git a/Makefile b/Makefile index 9e733491..77bc9b70 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -TARGETS := $(shell ls scripts | grep -vE 'clean|run') +TARGETS := $(shell ls scripts | grep -vE 'clean|run|help') .dapper: @echo Downloading dapper @@ -28,6 +28,9 @@ run: build/initrd/.id clean: @./scripts/clean -.DEFAULT_GOAL := ci +help: + @./scripts/help + +.DEFAULT_GOAL := default .PHONY: $(TARGETS) diff --git a/scripts/default b/scripts/default new file mode 100755 index 00000000..d17e8165 --- /dev/null +++ b/scripts/default @@ -0,0 +1,7 @@ +#!/bin/bash + +cd $(dirname $0) + +./build +./prepare +./package diff --git a/scripts/dev b/scripts/dev new file mode 100755 index 00000000..d743a519 --- /dev/null +++ b/scripts/dev @@ -0,0 +1,8 @@ +#!/bin/bash +# help: For development, creates iso, kernel, initrd gzip compressed + +cd $(dirname $0) + +./build +./prepare +COMPRESS="gzip -1" ROOTFS=0 ./package diff --git a/scripts/help b/scripts/help new file mode 100755 index 00000000..7f1f112e --- /dev/null +++ b/scripts/help @@ -0,0 +1,16 @@ +#!/bin/bash + +cd $(dirname $0) + +echo Targets: + +for i in *; do + if [ ! -x $i ] || [ ! -f $i ]; then + continue + fi + + MSG=$(grep '^# help:' $i) + if [ -n "$MSG" ]; then + echo " " ${i}: $(echo $MSG | sed 's/# help://') + fi +done diff --git a/scripts/integration-test b/scripts/integration-test new file mode 100755 index 00000000..741803e7 --- /dev/null +++ b/scripts/integration-test @@ -0,0 +1,11 @@ +#!/bin/bash +# help: Run Python based integration tests +set -e + +cd $(dirname $0)/../tests/integration + +if [ ! -e ../../dist/artifacts/initrd ]; then + ../../scripts/dev +fi + +tox "$@" diff --git a/scripts/package b/scripts/package index 99018d6a..324564c4 100755 --- a/scripts/package +++ b/scripts/package @@ -3,6 +3,8 @@ set -e cd $(dirname $0) -./package-rootfs +if [ "$ROOTFS" != "0" ]; then + ./package-rootfs +fi ./package-initrd ./package-iso diff --git a/scripts/package-initrd b/scripts/package-initrd index ce8ec456..84a6c84b 100755 --- a/scripts/package-initrd +++ b/scripts/package-initrd @@ -1,41 +1,21 @@ #!/bin/bash set -e -rootfs() { - DFS=$(docker run -d --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) ${DFS_IMAGE}${SUFFIX}) - trap "docker rm -fv ${DFS_ARCH} ${DFS}" EXIT - docker exec -i ${DFS} docker load < ${INITRD_DIR}/usr/share/ros/images.tar - docker stop ${DFS} - docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./image | tar -x -C ${PREPOP_DIR} - docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./overlay | tar -x -C ${PREPOP_DIR} - - tar -cf ${ARTIFACTS}/rootfs.tar --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} . - tar -rf ${ARTIFACTS}/rootfs.tar --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} . - rm -f ${ARTIFACTS}/rootfs.tar.gz - gzip ${ARTIFACTS}/rootfs.tar -} - -initrd() { - COMPRESS=lzma - [ "$DEV_BUILD" == "1" ] && COMPRESS="gzip -1" - - pushd ${INITRD_DIR} >/dev/null - - find | cpio -H newc -o | ${COMPRESS} > ${INITRD} - - popd >/dev/null -} - cd $(dirname $0)/.. BUILD=$(pwd)/build -IMAGE_CACHE=${BUILD}/image-cache -PREPOP_DIR=${IMAGE_CACHE}/var/lib/system-docker INITRD_DIR=${BUILD}/initrd ARTIFACTS=$(pwd)/dist/artifacts INITRD=${ARTIFACTS}/initrd -mkdir -p ${ARTIFACTS} ${PREPOP_DIR} +mkdir -p ${ARTIFACTS} -rootfs -initrd +if [ "$COMPRESS" == "" ]; then + COMPRESS=lzma +fi + +cd ${INITRD_DIR} + +echo Creating ${INITRD} +find | cpio -H newc -o | ${COMPRESS} > ${INITRD} +echo Done creating ${INITRD} diff --git a/scripts/package-rootfs b/scripts/package-rootfs index ce8ec456..6872d9ce 100755 --- a/scripts/package-rootfs +++ b/scripts/package-rootfs @@ -1,31 +1,6 @@ #!/bin/bash set -e -rootfs() { - DFS=$(docker run -d --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) ${DFS_IMAGE}${SUFFIX}) - trap "docker rm -fv ${DFS_ARCH} ${DFS}" EXIT - docker exec -i ${DFS} docker load < ${INITRD_DIR}/usr/share/ros/images.tar - docker stop ${DFS} - docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./image | tar -x -C ${PREPOP_DIR} - docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./overlay | tar -x -C ${PREPOP_DIR} - - tar -cf ${ARTIFACTS}/rootfs.tar --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} . - tar -rf ${ARTIFACTS}/rootfs.tar --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} . - rm -f ${ARTIFACTS}/rootfs.tar.gz - gzip ${ARTIFACTS}/rootfs.tar -} - -initrd() { - COMPRESS=lzma - [ "$DEV_BUILD" == "1" ] && COMPRESS="gzip -1" - - pushd ${INITRD_DIR} >/dev/null - - find | cpio -H newc -o | ${COMPRESS} > ${INITRD} - - popd >/dev/null -} - cd $(dirname $0)/.. BUILD=$(pwd)/build @@ -37,5 +12,14 @@ INITRD=${ARTIFACTS}/initrd mkdir -p ${ARTIFACTS} ${PREPOP_DIR} -rootfs -initrd +DFS=$(docker run -d --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) ${DFS_IMAGE}${SUFFIX}) +trap "docker rm -fv ${DFS_ARCH} ${DFS}" EXIT +docker exec -i ${DFS} docker load < ${INITRD_DIR}/usr/share/ros/images.tar +docker stop ${DFS} +docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./image | tar -x -C ${PREPOP_DIR} +docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./overlay | tar -x -C ${PREPOP_DIR} + +tar -cf ${ARTIFACTS}/rootfs.tar --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} . +tar -rf ${ARTIFACTS}/rootfs.tar --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} . +rm -f ${ARTIFACTS}/rootfs.tar.gz +gzip ${ARTIFACTS}/rootfs.tar diff --git a/scripts/prepare b/scripts/prepare index d717bbef..a4cbb7c1 100755 --- a/scripts/prepare +++ b/scripts/prepare @@ -5,7 +5,7 @@ source $(dirname $0)/version cd $(dirname $0) -if [ ! -e ./bin/host_ros ]; then +if [ ! -e ../bin/host_ros ]; then ./build fi diff --git a/scripts/run b/scripts/run index 2da83683..ab34044c 100755 --- a/scripts/run +++ b/scripts/run @@ -1,5 +1,6 @@ #!/bin/bash set -e +# help: build and run RancherOS, requires KVM local cd $(dirname $0)/.. @@ -108,7 +109,7 @@ if [[ ! -e ${KERNEL} || ! -e ${INITRD_SRC} ]]; then exit 1 fi -if [ "$REBUILD" == "1" ]; then +if [ "$REBUILD" == "1" ] || [ ! -e ${INITRD} ]; then cp bin/ros ${INITRD_SRC}/usr/bin/ros pushd ${INITRD_SRC} >/dev/null find . | cpio -H newc -o | gzip -1 > ${INITRD} @@ -157,6 +158,7 @@ if [ "$QEMU" == "1" ]; then CPU="-cpu host" fi set -x + HOME=${HOME:-/} exec qemu-system-${QEMUARCH} -serial stdio \ -rtc base=utc,clock=host \ ${KVM_ENABLE} \ diff --git a/scripts/shell b/scripts/shell new file mode 100755 index 00000000..811f0cfd --- /dev/null +++ b/scripts/shell @@ -0,0 +1,4 @@ +#!/bin/bash +# help: Launch shell in build environment + +exec bash diff --git a/scripts/test b/scripts/test index 4a6795b8..5af23658 100755 --- a/scripts/test +++ b/scripts/test @@ -1,4 +1,5 @@ #!/bin/bash +# help: Run go unit tests set -e cd $(dirname $0)/.. diff --git a/tests/integration/rostest/util.py b/tests/integration/rostest/util.py index e9be1a52..3d73ed56 100644 --- a/tests/integration/rostest/util.py +++ b/tests/integration/rostest/util.py @@ -8,7 +8,7 @@ import time import pytest ros_test = 'ros-test' -arch = os.environ['ARCH'] +arch = os.environ.get('ARCH', 'amd64') suffix = '' if arch != 'amd64':