dind-less, make-based build system

.drone.yml

@@ -1,3 +1,3 @@
-image: rancher/dind:v0.1.0
+image: rancher/dind:v0.5.0
 script:
  - ./scripts/ci

.gitignore

@@ -7,7 +7,6 @@
 /gopath
 .dockerfile
 *.swp
-Dockerfile
 /tests/integration/MANIFEST
 /tests/integration/.venv*
 /tests/integration/.tox

Dockerfile

@@ -0,0 +1,5 @@
+FROM ros-build-base
+
+ENV CONTAINED=1
+
+COPY . ./

Dockerfile.base

@@ -0,0 +1,30 @@
+FROM debian:jessie
+RUN apt-get update && \
+    apt-get -y dist-upgrade && \
+    apt-get -y install locales sudo vim less curl wget git rsync build-essential syslinux isolinux xorriso \
+        libblkid-dev libmount-dev libselinux1-dev
+RUN locale-gen en_US.UTF-8
+RUN curl -sSL https://get.docker.com/ | sh
+
+#ENV LANG en_US.UTF-8
+#ENV LANGUAGE en_US:en
+#ENV LC_ALL en_US.UTF-8
+#ENV TERM linux
+
+ENV GOLANG_VERSION 1.4.2
+RUN curl -sSL https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz | tar -v -C /usr/src -xz
+RUN cd /usr/src/go/src && ./make.bash --no-clean 2>&1
+
+ENV GOROOT /usr/src/go
+ENV PATH $GOROOT/bin:$PATH
+
+RUN go clean -i net
+RUN go install -tags netgo std
+
+RUN mkdir -p /go/src /go/bin && chmod -R 777 /go
+ENV GOPATH /go
+ENV PATH /go/bin:$PATH
+
+RUN go get github.com/tools/godep
+
+WORKDIR /go/src/github.com/rancherio/os

Makefile

@@ -0,0 +1,117 @@
+
+DOCKER_BINARY_URL := https://github.com/rancher/docker/releases/download/v1.7.1-ros-1/docker-1.7.1
+
+
+DOCKER_BINARY := $(shell basename $(DOCKER_BINARY_URL))
+pwd := $(shell pwd)
+include scripts/build-common
+include scripts/version
+
+
+compile: bin/rancheros
+
+all: clean ros-build-base package
+
+
+ros-build-base:
+	docker build -t ros-build-base -f Dockerfile.base .
+
+ros-build:
+	docker build -t ros-build .
+
+docker-run: ros-build
+	docker rm -fv ros-build > /dev/null 2>&1 || :
+	docker run -v /var/run/docker.sock:/var/run/docker.sock --name=ros-build -i ros-build
+
+
+CD := $(BUILD)/cd
+
+assets bin $(DIST)/artifacts $(CD)/boot/isolinux:
+	mkdir -p $@
+
+
+ifdef CONTAINED
+
+
+assets/$(DOCKER_BINARY): assets
+	cd assets && curl -OL "$(DOCKER_BINARY_URL)"
+
+assets/docker: assets/$(DOCKER_BINARY)
+	mv assets/$(DOCKER_BINARY) $@
+	chmod +x $@
+
+bin/rancheros: bin
+	godep go build -tags netgo -ldflags "-X github.com/rancherio/os/config.VERSION $(VERSION) -linkmode external -extldflags -static" -o $@
+	strip --strip-all $@
+
+copy-images:
+	./scripts/copy-images
+.PHONY: copy-images
+
+$(DIST)/artifacts/vmlinuz: $(DIST)/artifacts copy-images
+	mv $(BUILD)/kernel/vmlinuz $@
+
+
+INITRD_DIR := $(BUILD)/initrd
+
+$(INITRD_DIR)/images.tar: bin/rancheros
+	ln -sf bin/rancheros ./ros
+	for i in `./ros c images -i os-config.yml`; do docker pull $$i; done
+	docker save `./ros c images -i os-config.yml` > $@
+
+
+$(DIST)/artifacts/initrd: $(DIST)/artifacts bin/rancheros assets/docker copy-images $(INITRD_DIR)/images.tar
+	mv $(BUILD)/kernel/lib $(INITRD_DIR)
+	mv assets/docker       $(INITRD_DIR)
+	cp os-config.yml       $(INITRD_DIR)
+	cp bin/rancheros       $(INITRD_DIR)/init
+	cd $(INITRD_DIR) && find | cpio -H newc -o | lzma -c > $@
+
+$(DIST)/artifacts/rancheros.iso: $(DIST)/artifacts/initrd $(CD)/boot/isolinux
+	cp $(DIST)/artifacts/initrd                   $(CD)/boot
+	cp $(DIST)/artifacts/vmlinuz                  $(CD)/boot
+	cp scripts/isolinux.cfg                       $(CD)/boot/isolinux
+	cp /usr/lib/ISOLINUX/isolinux.bin             $(CD)/boot/isolinux
+	cp /usr/lib/syslinux/modules/bios/ldlinux.c32 $(CD)/boot/isolinux
+	cd $(CD) && xorriso -publisher "Rancher Labs, Inc." \
+		-as mkisofs \
+		-l -J -R -V "RancherOS" \
+		-no-emul-boot -boot-load-size 4 -boot-info-table \
+		-b boot/isolinux/isolinux.bin -c boot/isolinux/boot.cat \
+		-isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin \
+		-o $@ $(CD)
+
+$(DIST)/artifacts/iso-checksums.txt: $(DIST)/artifacts/rancheros.iso
+	cd $(DIST)/artifacts && for algo in 'sha256' 'md5'; do echo "$$algo: `$${algo}sum rancheros.iso`" >> $@; done
+
+package: \
+	$(DIST)/artifacts/initrd \
+	$(DIST)/artifacts/vmlinuz \
+	$(DIST)/artifacts/rancheros.iso \
+	$(DIST)/artifacts/iso-checksums.txt
+
+
+else
+
+
+bin/rancheros:
+	@echo make $@ | make docker-run
+	docker cp ros-build:/go/src/github.com/rancherio/os/$@ $(dir $@)
+.PHONY: bin/rancheros
+
+package:
+	@echo make $@ | make docker-run
+	docker cp ros-build:/go/src/github.com/rancherio/os/bin/rancheros bin
+	docker cp ros-build:/go/src/github.com/rancherio/os/dist/artifacts dist
+
+
+endif
+
+
+version:
+	@echo $(VERSION)
+
+clean:
+	rm -rf bin build dist gopath .dockerfile
+
+.PHONY: all compile clean dist docker-run download package ros-build ros-build-base version

build.sh

@@ -1,21 +1,6 @@
 #!/bin/bash
 set -e
 
-cd $(dirname $0)
+make all
 
-export DOCKER_IMAGE=rancher-os-build
+ls -lh dist/artifacts
-
-./scripts/ci
-
-rm -rf dist
-
-echo "Build complete. Copying artifacts..."
-DIST_CONTAINER=$(docker create ${DOCKER_IMAGE})
-cleanup() {
-    docker rm -v ${DIST_CONTAINER}
-}
-trap cleanup EXIT
-docker cp ${DIST_CONTAINER}:/source/dist/artifacts dist
-docker cp ${DIST_CONTAINER}:/source/bin ./
-
-ls -l dist/artifacts

scripts/bootstrap

@@ -1,11 +0,0 @@
-#!/bin/bash
-set -e
-set -x
-
-cd $(dirname $0)/..
-
-apt-get update
-apt-get install -y curl rsync build-essential syslinux xorriso libblkid-dev libmount-dev libselinux1-dev
-
-go clean -i net
-go install -tags netgo std

scripts/build

@@ -1,58 +0,0 @@
-#!/bin/bash -e
-set -x
-
-cd $(dirname $0)/..
-
-source scripts/version
-
-while [ "$#" -gt 0 ]; do
-    case $1 in
-        --custom)
-            IN_DOCKER=true
-            ROS_CUSTOM_GOPATH=true
-            ;;
-        *)
-            break
-            ;;
-    esac
-    shift 1
-done
-
-if [ "$IN_DOCKER" != "true" ]; then
-    cat > .dockerfile << "EOF"
-FROM rancher/dind:v0.5.0
-COPY ./scripts/bootstrap /scripts/bootstrap
-RUN /scripts/bootstrap
-WORKDIR /source
-ENV IN_DOCKER true
-EOF
-
-    docker build -t rancher-os-go-build -f .dockerfile .
-    docker run --rm -v $(pwd):/source -it rancher-os-go-build ./scripts/build
-    exit 0
-fi
-
-if [[ ! -x "$(which go)" && -x /usr/local/go/bin/go ]]; then
-    PATH=/usr/local/go/bin:${PATH}
-fi
-
-if [ -z "$ROS_CUSTOM_GOPATH" ]; then
-    export GOPATH=$(pwd)/Godeps/_workspace:$(pwd)/gopath
-fi
-
-PACKAGE=./gopath/src/$(<.package)
-
-if [ -L ${PACKAGE} ]; then
-    rm ${PACKAGE}
-fi
-
-if [ ! -e ${PACKAGE} ]; then
-    mkdir -p $(dirname $PACKAGE)
-    ln -s $(pwd) $PACKAGE
-fi
-
-echo export GOPATH=$GOPATH
-
-mkdir -p bin
-go build -tags netgo -ldflags "-X github.com/rancherio/os/config.VERSION ${VERSION:-v0.0.0-dev} -linkmode external -extldflags -static" -o bin/rancheros
-strip --strip-all bin/rancheros

scripts/build-common

@@ -1,131 +1,3 @@
-#!/bin/bash
+ARTIFACTS=$(pwd)/assets
-
+BUILD=$(pwd)/build
-: ${ARTIFACTS:=$(pwd)/assets}
+DIST=$(pwd)/dist
-: ${BUILD:=$(pwd)/build}
-: ${DIST:=$(pwd)/dist}
-
-# PREREQ: brew install coreutils
-path()
-{
-    local UNAME=$(uname)
-    if [ "$UNAME" == "Darwin" ]; then greadlink -f "$1"
-    elif [ "$UNAME" == "Linux" ]; then readlink -f "$1";
-    fi
-}
-
-write_base()
-{
-    DOCKER_BASE=${DOCKER_BASE:?"DOCKER_BASE not defined"}
-
-    if [ "${BASE_WRITTEN}" = "true" ]; then
-        return
-    fi
-
-    mkdir -p $(dirname ${DOCKER_FILE})
-
-    cat > ${DOCKER_FILE} << EOF
-FROM ${DOCKER_BASE}
-ENV TERM xterm
-ENV IN_DOCKER true
-WORKDIR /source
-EOF
-
-    BASE_WRITTEN=true
-}
-
-run()
-{
-    local content
-
-    while [ $# -gt 1 ]; do
-        case $1 in
-            --assets)
-                shift 1
-                if [ -e "$1" ]; then
-                    content="$content\nCOPY $1 /source/$1"
-                else
-                    content="$content\nCOPY $1 /source/"
-                fi
-                ;;
-            *)
-                break
-                ;;
-        esac
-
-        shift 1
-    done
-
-    write_base
-    if [ -n "$content" ]; then
-        echo -e "$content" >> ${DOCKER_FILE}
-    fi
-    if [ -n "$1" ]; then
-        echo -e "\nCOPY $1 /source/$1" >> ${DOCKER_FILE}
-        echo -e "RUN /source/"$@"" >> ${DOCKER_FILE}
-    fi
-
-    if [ "$RUN_EXEC" = "true" ]; then
-        "$@"
-    fi
-}
-
-finish()
-{
-    if [ "$RUN_EXEC" = "true" ]; then
-        return
-    fi
-
-    local cmd="docker build -t ${DOCKER_IMAGE} -f ${DOCKER_FILE} ."
-    echo Running $cmd
-    echo Pwd $(pwd)
-
-    cat ${DOCKER_FILE}
-
-    $cmd
-}
-
-reset_docker_build()
-{
-    BASE_WRITTEN=false
-}
-
-check()
-{
-    local hash=$1
-    local file=$2
-
-    if [ ! -e "$file" ]; then
-        return 1
-    fi
-
-    if [ "$hash" = dev ]; then
-        return 0
-    fi
-
-    CURRENT=$(sha1sum $file | awk '{print $1}')
-
-    [ "$hash" = "$CURRENT" ]
-}
-
-download()
-{
-    mkdir -p ${ARTIFACTS}
-
-    local url=$2
-    local file=${ARTIFACTS}/$(basename $2)
-    local hash=$1
-
-    if [ "$3" != "" ]; then
-        file=${ARTIFACTS}/$3
-    fi
-
-    if ! check $hash $file; then
-        echo Downloading $url to $file
-        curl -sL $url > $file
-    fi
-
-    if ! check $hash $file; then
-        echo "ERROR: $file does not match checksum $hash, got $CURRENT" 1>&2
-        return 1
-    fi
-}

scripts/build-images

@@ -1,48 +0,0 @@
-#!/bin/bash
-set -ex
-
-cd $(dirname $0)/..
-
-source scripts/build-common
-source scripts/version
-ARTIFACTS=${ARTIFACTS:?"ARTIFACTS not set"}
-VERSION=${VERSION:?"VERSION not set"}
-BUILD=${BUILD:?"BUILD not set"}
-DIST=${DIST:?"DIST not set"}
-
-if [ -x "$(which wrapdocker)" ] && ! docker info >/dev/null 2>&1; then
-    wrapdocker
-fi
-
-CONTAINER_INITRDBASE=$(docker create rancher/os-initrdbase:${VERSION})
-cleanup_initrdbase() {
-    docker rm -v ${CONTAINER_INITRDBASE}
-}
-trap cleanup_initrdbase EXIT
-docker cp ${CONTAINER_INITRDBASE}:/initrd ${BUILD} # copies files to ${BUILD}/initrd
-
-INITRD_DIR=${BUILD}/initrd
-
-
-mkdir -p ${DIST}/artifacts
-
-CONTAINER_KERNEL=$(docker create rancher/os-kernel:${VERSION})
-cleanup_kernel() {
-    docker rm -v ${CONTAINER_KERNEL}
-}
-trap cleanup_kernel EXIT
-docker cp ${CONTAINER_KERNEL}:/kernel ${BUILD} # copies files to ${BUILD}/kernel
-mv ${BUILD}/kernel/vmlinuz ${DIST}/artifacts/                               #kernel
-
-
-mv ${BUILD}/kernel/lib ${INITRD_DIR}                                        #initrd2-kernel: /lib
-
-cp -f ${ARTIFACTS}/docker* ${INITRD_DIR}/docker                             #initrd2-docker: /docker
-chmod +x ${INITRD_DIR}/docker                                               #initrd2-docker: /docker
-
-ln -sf bin/rancheros ./ros
-
-for i in $(./ros c images -i os-config.yml); do
-    docker pull ${i}
-done
-docker save $(./ros c images -i os-config.yml) > ${INITRD_DIR}/images.tar   #initrd2-images: /images.tar

scripts/ci

@@ -3,57 +3,10 @@ set -ex
 
 cd $(dirname $0)/..
 
-export DOCKER_IMAGE=${DOCKER_IMAGE:=rancher-os-build}
-export DOCKER_BASE=rancher/dind:v0.5.0
-
-source scripts/build-common
-mkdir -p ${BUILD}
-
-DOCKER_FILE=$(pwd)/.dockerfile
-
-generate_images()
-{
-    if [ "$RUN_EXEC" = "true" ]; then
-        ./scripts/build-images
-        return
-    fi
-
-    IMAGE_ID=$(docker images --no-trunc -q ${DOCKER_IMAGE})
-
-    if [ -e ${BUILD}/${IMAGE_ID} ]; then
-        DOCKER_BASE=$(<${BUILD}/${IMAGE_ID})
-    else
-        echo Running: docker run -d --privileged ${DOCKER_IMAGE} /source/scripts/build-images
-        CID=$(docker run -d --privileged ${DOCKER_IMAGE} /source/scripts/build-images)
-        docker logs -f ${CID} &
-        trap "docker rm -fv ${CID}" exit
-        [ "$(docker wait $CID)" == 0 ]
-        DOCKER_BASE=$(docker commit $CID)
-
-        echo ${DOCKER_BASE} > ${BUILD}/${IMAGE_ID}
-    fi
-}
-
 if [ -x "$(which wrapdocker)" ] && ! docker info >/dev/null 2>&1; then
     wrapdocker
 fi
 
-run ./scripts/bootstrap
+docker build -t ros-build-base -f Dockerfile.base .
-ARGS=
+docker build -t ros-build .
-for i in $(ls -d * .* | sort -u | grep -Ev '(\.|\.\.|\.dockerfile|build|tmp|dist|.DS_Store|.git|.idea|.vagrant|scripts|bin|state)$'); do
+echo make package | docker run -v /var/run/docker.sock:/var/run/docker.sock --name=ros-build -i ros-build
-    if [ -d $i ]; then
-        run --assets $i
-    else
-        ARGS="${ARGS} $i"
-    fi
-done
-run --assets ./scripts/version --assets "${ARGS}" ./scripts/build
-run --assets ./scripts/build-common --assets ./assets ./scripts/download
-run --assets ./scripts/build-images
-finish
-
-generate_images
-
-reset_docker_build
-run ./scripts/package "$@"
-finish

scripts/clean

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cd $(dirname $0)/..
-
-rm -rf build dist gopath .dockerfile Dockerfile

scripts/copy-images

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -ex
+
+cd $(dirname $0)/..
+
+. scripts/build-common
+. scripts/version
+
+VERSION=${VERSION:?"VERSION not set"}
+BUILD=${BUILD:?"BUILD not set"}
+
+
+CONTAINER_INITRDBASE=$(docker create rancher/os-initrdbase:${VERSION})
+cleanup_initrdbase() {
+    docker rm -v ${CONTAINER_INITRDBASE}
+}
+trap cleanup_initrdbase EXIT
+docker cp ${CONTAINER_INITRDBASE}:/initrd ${BUILD} # copies files to ${BUILD}/initrd
+
+
+CONTAINER_KERNEL=$(docker create rancher/os-kernel:${VERSION})
+cleanup_kernel() {
+    docker rm -v ${CONTAINER_KERNEL}
+}
+trap cleanup_kernel EXIT
+docker cp ${CONTAINER_KERNEL}:/kernel ${BUILD} # copies files to ${BUILD}/kernel

scripts/download

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-cd $(dirname $0)/..
-
-source scripts/build-common
-
-download 99652ec62c682041b67aa917d9b040d5541b4e78 https://github.com/rancher/docker/releases/download/v1.7.1-ros-1/docker-1.7.1

scripts/isolinux.cfg

@@ -0,0 +1,5 @@
+default rancheros
+label rancheros
+    kernel /boot/vmlinuz
+    initrd /boot/initrd
+    append quiet rancher.password=rancher rancher.state.autoformat=[/dev/sda,/dev/vda]

scripts/package

@@ -1,56 +0,0 @@
-#!/bin/bash
-set -ex
-
-cd $(dirname $0)/..
-
-source scripts/build-common
-
-cp bin/rancheros ${BUILD}/initrd/init
-cp ./os-config.yml ${BUILD}/initrd/
-
-cd ${BUILD}/initrd
-
-if [ "--dev" = "$1" ]; then
-    find | cpio -H newc -o > ${DIST}/artifacts/initrd
-    cp ${DIST}/artifacts/initrd ${DIST}/artifacts/initrd.none
-else
-    find | cpio -H newc -o | lzma -c > ${DIST}/artifacts/initrd
-fi
-
-CD=${BUILD}/cd
-
-mkdir -p ${CD}/boot/isolinux
-cp ${DIST}/artifacts/vmlinuz ${CD}/boot
-cp ${DIST}/artifacts/initrd ${CD}/boot
-cp /usr/lib/syslinux/isolinux.bin ${CD}/boot/isolinux
-cp /usr/lib/syslinux/linux.c32 ${CD}/boot/isolinux/ldlinux.c32
-
-cat > ${CD}/boot/isolinux/isolinux.cfg << EOF
-default rancheros
-label rancheros
-    kernel /boot/vmlinuz
-    initrd /boot/initrd
-    append quiet rancher.password=rancher rancher.state.autoformat=[/dev/sda,/dev/vda]
-EOF
-
-cd ${CD}
-# Copied from boot2docker, thanks.
-xorriso \
-    -publisher "Rancher Labs, Inc." \
-    -as mkisofs \
-    -l -J -R -V "RancherOS" \
-    -no-emul-boot -boot-load-size 4 -boot-info-table \
-    -b boot/isolinux/isolinux.bin -c boot/isolinux/boot.cat \
-    -isohybrid-mbr /usr/lib/syslinux/isohdpfx.bin \
-    -o ${DIST}/artifacts/rancheros.iso $(pwd)
-
-
-if [ -e ${DIST}/artficats/iso-checksums.txt ]; then
-    rm ${DIST}/artficats/iso-checksums.txt
-fi
-
-for algorithm in 'sha256' 'md5'; do
-    for i in ${DIST}/artifacts/*iso; do
-        echo "${algorithm}: $(${algorithm}sum ${i})" >> ${DIST}/artifacts/iso-checksums.txt
-    done
-done

scripts/release

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-if ! docker info >/dev/null 2>&1 && [ -x "$(which wrapdocker)" ]; then
-    wrapdocker
-fi
-
-exec $(dirname $0)/../build.sh

scripts/run

@@ -18,6 +18,14 @@ INITRD_CURRENT=${BUILD}/initrd-current
 INITRD_TEST=${BUILD}/initrd.test
 USER_DATA=cloud-init/openstack/latest/user_data
 
+# PREREQ: brew install coreutils
+path() {
+    local UNAME=$(uname)
+    if [ "$UNAME" == "Darwin" ]; then greadlink -f "$1"
+    elif [ "$UNAME" == "Linux" ]; then readlink -f "$1";
+    fi
+}
+
 while [ "$#" -gt 0 ]; do
     case $1 in
         --append)