Speed up build and run tests

.dockerignore

@@ -4,6 +4,7 @@ bin
 state
 build
 images/*/build
+scripts/images/*/dist/
 dist
 tests/integration/.venv*
 tests/integration/.tox

Dockerfile.dapper

@@ -28,6 +28,7 @@ RUN apt-get update && \
         wget \
         xorriso
 
+ENV DAPPER_ENV VERSION DEV_BUILD
 ENV DAPPER_DOCKER_SOCKET true
 ENV DAPPER_SOURCE /go/src/github.com/rancher/os
 ENV DAPPER_OUTPUT ./bin ./dist ./build/initrd
@@ -99,10 +100,12 @@ RUN if [ -n "${!VBOX_MODULES_URL}" ]; then \
     ;fi
 
 # Install Go
+ENV GO_VERSION 1.6.2
 RUN ln -sf go-6 /usr/bin/go && \
-    curl -sfL https://storage.googleapis.com/golang/go1.6.src.tar.gz | tar -xzf - -C /usr/local && \
+    curl -sfL https://storage.googleapis.com/golang/go${GO_VERSION}.src.tar.gz | tar -xzf - -C /usr/local && \
     cd /usr/local/go/src && \
-    GOROOT_BOOTSTRAP=/usr GOARCH=${HOST_ARCH} GOHOSTARCH=${HOST_ARCH} ./make.bash
+    GOROOT_BOOTSTRAP=/usr GOARCH=${HOST_ARCH} GOHOSTARCH=${HOST_ARCH} ./make.bash && \
+    rm /usr/bin/go
 ENV GOPATH /go
 ENV PATH ${GOPATH}/bin:/usr/local/go/bin:$PATH
 

Makefile

@@ -1,4 +1,4 @@
-TARGETS := $(shell ls scripts | grep -vE 'clean|run')
+TARGETS := $(shell ls scripts | grep -vE 'clean|run|help')
 
 .dapper:
 	@echo Downloading dapper
@@ -28,6 +28,9 @@ run: build/initrd/.id
 clean:
 	@./scripts/clean
 
-.DEFAULT_GOAL := ci
+help:
+	@./scripts/help
+
+.DEFAULT_GOAL := default
 
 .PHONY: $(TARGETS)

scripts/default

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cd $(dirname $0)
+
+./build
+./prepare
+./package

scripts/dev

@@ -0,0 +1,8 @@
+#!/bin/bash
+# help: For development, creates iso, kernel, initrd gzip compressed
+
+cd $(dirname $0)
+
+./build
+./prepare
+COMPRESS="gzip -1" ROOTFS=0 ./package

scripts/help

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd $(dirname $0)
+
+echo Targets:
+
+for i in *; do
+    if [ ! -x $i ] || [ ! -f $i ]; then
+        continue
+    fi
+
+    MSG=$(grep '^# help:' $i)
+    if [ -n "$MSG" ]; then
+        echo "   " ${i}: $(echo $MSG | sed 's/# help://')
+    fi
+done

scripts/integration-test

@@ -0,0 +1,11 @@
+#!/bin/bash
+# help: Run Python based integration tests
+set -e
+
+cd $(dirname $0)/../tests/integration
+
+if [ ! -e ../../dist/artifacts/initrd ]; then
+    ../../scripts/dev
+fi
+
+tox "$@"

scripts/package

@@ -3,6 +3,8 @@ set -e
 
 cd $(dirname $0)
 
+if [ "$ROOTFS" != "0" ]; then
     ./package-rootfs
+fi
 ./package-initrd
 ./package-iso

scripts/package-initrd

@@ -1,41 +1,21 @@
 #!/bin/bash
 set -e
 
-rootfs() {
-  DFS=$(docker run -d --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) ${DFS_IMAGE}${SUFFIX})
-  trap "docker rm -fv ${DFS_ARCH} ${DFS}" EXIT
-  docker exec -i ${DFS} docker load < ${INITRD_DIR}/usr/share/ros/images.tar
-  docker stop ${DFS}
-  docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./image | tar -x -C ${PREPOP_DIR}
-  docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./overlay | tar -x -C ${PREPOP_DIR}
-
-  tar -cf ${ARTIFACTS}/rootfs.tar  --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} .
-  tar -rf ${ARTIFACTS}/rootfs.tar  --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} .
-  rm -f ${ARTIFACTS}/rootfs.tar.gz
-  gzip ${ARTIFACTS}/rootfs.tar
-}
-
-initrd() {
-  COMPRESS=lzma
-  [ "$DEV_BUILD" == "1" ] && COMPRESS="gzip -1"
-
-  pushd ${INITRD_DIR} >/dev/null
-
-  find | cpio -H newc -o | ${COMPRESS} > ${INITRD}
-
-  popd >/dev/null
-}
-
 cd $(dirname $0)/..
 
 BUILD=$(pwd)/build
-IMAGE_CACHE=${BUILD}/image-cache
-PREPOP_DIR=${IMAGE_CACHE}/var/lib/system-docker
 INITRD_DIR=${BUILD}/initrd
 ARTIFACTS=$(pwd)/dist/artifacts
 INITRD=${ARTIFACTS}/initrd
 
-mkdir -p ${ARTIFACTS} ${PREPOP_DIR}
+mkdir -p ${ARTIFACTS}
 
-rootfs
+if [ "$COMPRESS" == "" ]; then
-initrd
+    COMPRESS=lzma
+fi
+
+cd ${INITRD_DIR}
+
+echo Creating ${INITRD}
+find | cpio -H newc -o | ${COMPRESS} > ${INITRD}
+echo Done creating ${INITRD}

scripts/package-rootfs

@@ -1,31 +1,6 @@
 #!/bin/bash
 set -e
 
-rootfs() {
-  DFS=$(docker run -d --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) ${DFS_IMAGE}${SUFFIX})
-  trap "docker rm -fv ${DFS_ARCH} ${DFS}" EXIT
-  docker exec -i ${DFS} docker load < ${INITRD_DIR}/usr/share/ros/images.tar
-  docker stop ${DFS}
-  docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./image | tar -x -C ${PREPOP_DIR}
-  docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./overlay | tar -x -C ${PREPOP_DIR}
-
-  tar -cf ${ARTIFACTS}/rootfs.tar  --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} .
-  tar -rf ${ARTIFACTS}/rootfs.tar  --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} .
-  rm -f ${ARTIFACTS}/rootfs.tar.gz
-  gzip ${ARTIFACTS}/rootfs.tar
-}
-
-initrd() {
-  COMPRESS=lzma
-  [ "$DEV_BUILD" == "1" ] && COMPRESS="gzip -1"
-
-  pushd ${INITRD_DIR} >/dev/null
-
-  find | cpio -H newc -o | ${COMPRESS} > ${INITRD}
-
-  popd >/dev/null
-}
-
 cd $(dirname $0)/..
 
 BUILD=$(pwd)/build
@@ -37,5 +12,14 @@ INITRD=${ARTIFACTS}/initrd
 
 mkdir -p ${ARTIFACTS} ${PREPOP_DIR}
 
-rootfs
+DFS=$(docker run -d --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) ${DFS_IMAGE}${SUFFIX})
-initrd
+trap "docker rm -fv ${DFS_ARCH} ${DFS}" EXIT
+docker exec -i ${DFS} docker load < ${INITRD_DIR}/usr/share/ros/images.tar
+docker stop ${DFS}
+docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./image | tar -x -C ${PREPOP_DIR}
+docker run --rm --volumes-from=${DFS} rancher/os-dapper-base tar -c -C /var/lib/docker ./overlay | tar -x -C ${PREPOP_DIR}
+
+tar -cf ${ARTIFACTS}/rootfs.tar  --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} .
+tar -rf ${ARTIFACTS}/rootfs.tar  --exclude lib/modules --exclude lib/firmware -C ${INITRD_DIR} .
+rm -f ${ARTIFACTS}/rootfs.tar.gz
+gzip ${ARTIFACTS}/rootfs.tar

scripts/prepare

@@ -5,7 +5,7 @@ source $(dirname $0)/version
 
 cd $(dirname $0)
 
-if [ ! -e ./bin/host_ros ]; then
+if [ ! -e ../bin/host_ros ]; then
     ./build
 fi
 

scripts/run

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -e
+# help: build and run RancherOS, requires KVM local
 
 cd $(dirname $0)/..
 
@@ -108,7 +109,7 @@ if [[ ! -e ${KERNEL} || ! -e ${INITRD_SRC} ]]; then
     exit 1
 fi
 
-if [ "$REBUILD" == "1" ]; then
+if [ "$REBUILD" == "1" ] || [ ! -e ${INITRD} ]; then
     cp bin/ros ${INITRD_SRC}/usr/bin/ros
     pushd ${INITRD_SRC} >/dev/null
     find . | cpio -H newc -o | gzip -1 > ${INITRD}
@@ -157,6 +158,7 @@ if [ "$QEMU" == "1" ]; then
         CPU="-cpu host"
     fi
     set -x
+    HOME=${HOME:-/}
     exec qemu-system-${QEMUARCH} -serial stdio \
         -rtc base=utc,clock=host \
         ${KVM_ENABLE} \

scripts/shell

@@ -0,0 +1,4 @@
+#!/bin/bash
+# help: Launch shell in build environment
+
+exec bash

scripts/test

@@ -1,4 +1,5 @@
 #!/bin/bash
+# help: Run go unit tests
 set -e
 
 cd $(dirname $0)/..

tests/integration/rostest/util.py

@@ -8,7 +8,7 @@ import time
 import pytest
 
 ros_test = 'ros-test'
-arch = os.environ['ARCH']
+arch = os.environ.get('ARCH', 'amd64')
 
 suffix = ''
 if arch != 'amd64':