diff --git a/.drone.yml b/.drone.yml index 51807b2c..d99f3244 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,3 @@ -image: rancher/dind:v0.1.0 +image: rancher/dind:v0.5.0 script: - ./scripts/ci diff --git a/.gitignore b/.gitignore index 36d864f9..342b3516 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ /gopath .dockerfile *.swp -Dockerfile /tests/integration/MANIFEST /tests/integration/.venv* /tests/integration/.tox diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d0389b28 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM ros-build-base + +ENV CONTAINED=1 + +COPY . ./ diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 00000000..3802b489 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,30 @@ +FROM debian:jessie +RUN apt-get update && \ + apt-get -y dist-upgrade && \ + apt-get -y install locales sudo vim less curl wget git rsync build-essential syslinux isolinux xorriso \ + libblkid-dev libmount-dev libselinux1-dev +RUN locale-gen en_US.UTF-8 +RUN curl -sSL https://get.docker.com/ | sh + +#ENV LANG en_US.UTF-8 +#ENV LANGUAGE en_US:en +#ENV LC_ALL en_US.UTF-8 +#ENV TERM linux + +ENV GOLANG_VERSION 1.4.2 +RUN curl -sSL https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz | tar -v -C /usr/src -xz +RUN cd /usr/src/go/src && ./make.bash --no-clean 2>&1 + +ENV GOROOT /usr/src/go +ENV PATH $GOROOT/bin:$PATH + +RUN go clean -i net +RUN go install -tags netgo std + +RUN mkdir -p /go/src /go/bin && chmod -R 777 /go +ENV GOPATH /go +ENV PATH /go/bin:$PATH + +RUN go get github.com/tools/godep + +WORKDIR /go/src/github.com/rancherio/os diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..49229333 --- /dev/null +++ b/Makefile @@ -0,0 +1,117 @@ + +DOCKER_BINARY_URL := https://github.com/rancher/docker/releases/download/v1.7.1-ros-1/docker-1.7.1 + + +DOCKER_BINARY := $(shell basename $(DOCKER_BINARY_URL)) +pwd := $(shell pwd) +include scripts/build-common +include scripts/version + + +compile: bin/rancheros + +all: clean ros-build-base package + + +ros-build-base: + docker build -t ros-build-base -f Dockerfile.base . + +ros-build: + docker build -t ros-build . + +docker-run: ros-build + docker rm -fv ros-build > /dev/null 2>&1 || : + docker run -v /var/run/docker.sock:/var/run/docker.sock --name=ros-build -i ros-build + + +CD := $(BUILD)/cd + +assets bin $(DIST)/artifacts $(CD)/boot/isolinux: + mkdir -p $@ + + +ifdef CONTAINED + + +assets/$(DOCKER_BINARY): assets + cd assets && curl -OL "$(DOCKER_BINARY_URL)" + +assets/docker: assets/$(DOCKER_BINARY) + mv assets/$(DOCKER_BINARY) $@ + chmod +x $@ + +bin/rancheros: bin + godep go build -tags netgo -ldflags "-X github.com/rancherio/os/config.VERSION $(VERSION) -linkmode external -extldflags -static" -o $@ + strip --strip-all $@ + +copy-images: + ./scripts/copy-images +.PHONY: copy-images + +$(DIST)/artifacts/vmlinuz: $(DIST)/artifacts copy-images + mv $(BUILD)/kernel/vmlinuz $@ + + +INITRD_DIR := $(BUILD)/initrd + +$(INITRD_DIR)/images.tar: bin/rancheros + ln -sf bin/rancheros ./ros + for i in `./ros c images -i os-config.yml`; do docker pull $$i; done + docker save `./ros c images -i os-config.yml` > $@ + + +$(DIST)/artifacts/initrd: $(DIST)/artifacts bin/rancheros assets/docker copy-images $(INITRD_DIR)/images.tar + mv $(BUILD)/kernel/lib $(INITRD_DIR) + mv assets/docker $(INITRD_DIR) + cp os-config.yml $(INITRD_DIR) + cp bin/rancheros $(INITRD_DIR)/init + cd $(INITRD_DIR) && find | cpio -H newc -o | lzma -c > $@ + +$(DIST)/artifacts/rancheros.iso: $(DIST)/artifacts/initrd $(CD)/boot/isolinux + cp $(DIST)/artifacts/initrd $(CD)/boot + cp $(DIST)/artifacts/vmlinuz $(CD)/boot + cp scripts/isolinux.cfg $(CD)/boot/isolinux + cp /usr/lib/ISOLINUX/isolinux.bin $(CD)/boot/isolinux + cp /usr/lib/syslinux/modules/bios/ldlinux.c32 $(CD)/boot/isolinux + cd $(CD) && xorriso -publisher "Rancher Labs, Inc." \ + -as mkisofs \ + -l -J -R -V "RancherOS" \ + -no-emul-boot -boot-load-size 4 -boot-info-table \ + -b boot/isolinux/isolinux.bin -c boot/isolinux/boot.cat \ + -isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin \ + -o $@ $(CD) + +$(DIST)/artifacts/iso-checksums.txt: $(DIST)/artifacts/rancheros.iso + cd $(DIST)/artifacts && for algo in 'sha256' 'md5'; do echo "$$algo: `$${algo}sum rancheros.iso`" >> $@; done + +package: \ + $(DIST)/artifacts/initrd \ + $(DIST)/artifacts/vmlinuz \ + $(DIST)/artifacts/rancheros.iso \ + $(DIST)/artifacts/iso-checksums.txt + + +else + + +bin/rancheros: + @echo make $@ | make docker-run + docker cp ros-build:/go/src/github.com/rancherio/os/$@ $(dir $@) +.PHONY: bin/rancheros + +package: + @echo make $@ | make docker-run + docker cp ros-build:/go/src/github.com/rancherio/os/bin/rancheros bin + docker cp ros-build:/go/src/github.com/rancherio/os/dist/artifacts dist + + +endif + + +version: + @echo $(VERSION) + +clean: + rm -rf bin build dist gopath .dockerfile + +.PHONY: all compile clean dist docker-run download package ros-build ros-build-base version diff --git a/build.sh b/build.sh index ada930b5..454369be 100755 --- a/build.sh +++ b/build.sh @@ -1,21 +1,6 @@ #!/bin/bash set -e -cd $(dirname $0) +make all -export DOCKER_IMAGE=rancher-os-build - -./scripts/ci - -rm -rf dist - -echo "Build complete. Copying artifacts..." -DIST_CONTAINER=$(docker create ${DOCKER_IMAGE}) -cleanup() { - docker rm -v ${DIST_CONTAINER} -} -trap cleanup EXIT -docker cp ${DIST_CONTAINER}:/source/dist/artifacts dist -docker cp ${DIST_CONTAINER}:/source/bin ./ - -ls -l dist/artifacts +ls -lh dist/artifacts diff --git a/scripts/bootstrap b/scripts/bootstrap deleted file mode 100755 index d8bffb32..00000000 --- a/scripts/bootstrap +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e -set -x - -cd $(dirname $0)/.. - -apt-get update -apt-get install -y curl rsync build-essential syslinux xorriso libblkid-dev libmount-dev libselinux1-dev - -go clean -i net -go install -tags netgo std diff --git a/scripts/build b/scripts/build deleted file mode 100755 index 8c9b923c..00000000 --- a/scripts/build +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -e -set -x - -cd $(dirname $0)/.. - -source scripts/version - -while [ "$#" -gt 0 ]; do - case $1 in - --custom) - IN_DOCKER=true - ROS_CUSTOM_GOPATH=true - ;; - *) - break - ;; - esac - shift 1 -done - -if [ "$IN_DOCKER" != "true" ]; then - cat > .dockerfile << "EOF" -FROM rancher/dind:v0.5.0 -COPY ./scripts/bootstrap /scripts/bootstrap -RUN /scripts/bootstrap -WORKDIR /source -ENV IN_DOCKER true -EOF - - docker build -t rancher-os-go-build -f .dockerfile . - docker run --rm -v $(pwd):/source -it rancher-os-go-build ./scripts/build - exit 0 -fi - -if [[ ! -x "$(which go)" && -x /usr/local/go/bin/go ]]; then - PATH=/usr/local/go/bin:${PATH} -fi - -if [ -z "$ROS_CUSTOM_GOPATH" ]; then - export GOPATH=$(pwd)/Godeps/_workspace:$(pwd)/gopath -fi - -PACKAGE=./gopath/src/$(<.package) - -if [ -L ${PACKAGE} ]; then - rm ${PACKAGE} -fi - -if [ ! -e ${PACKAGE} ]; then - mkdir -p $(dirname $PACKAGE) - ln -s $(pwd) $PACKAGE -fi - -echo export GOPATH=$GOPATH - -mkdir -p bin -go build -tags netgo -ldflags "-X github.com/rancherio/os/config.VERSION ${VERSION:-v0.0.0-dev} -linkmode external -extldflags -static" -o bin/rancheros -strip --strip-all bin/rancheros diff --git a/scripts/build-common b/scripts/build-common index 9dd3aa73..182c1b56 100644 --- a/scripts/build-common +++ b/scripts/build-common @@ -1,131 +1,3 @@ -#!/bin/bash - -: ${ARTIFACTS:=$(pwd)/assets} -: ${BUILD:=$(pwd)/build} -: ${DIST:=$(pwd)/dist} - -# PREREQ: brew install coreutils -path() -{ - local UNAME=$(uname) - if [ "$UNAME" == "Darwin" ]; then greadlink -f "$1" - elif [ "$UNAME" == "Linux" ]; then readlink -f "$1"; - fi -} - -write_base() -{ - DOCKER_BASE=${DOCKER_BASE:?"DOCKER_BASE not defined"} - - if [ "${BASE_WRITTEN}" = "true" ]; then - return - fi - - mkdir -p $(dirname ${DOCKER_FILE}) - - cat > ${DOCKER_FILE} << EOF -FROM ${DOCKER_BASE} -ENV TERM xterm -ENV IN_DOCKER true -WORKDIR /source -EOF - - BASE_WRITTEN=true -} - -run() -{ - local content - - while [ $# -gt 1 ]; do - case $1 in - --assets) - shift 1 - if [ -e "$1" ]; then - content="$content\nCOPY $1 /source/$1" - else - content="$content\nCOPY $1 /source/" - fi - ;; - *) - break - ;; - esac - - shift 1 - done - - write_base - if [ -n "$content" ]; then - echo -e "$content" >> ${DOCKER_FILE} - fi - if [ -n "$1" ]; then - echo -e "\nCOPY $1 /source/$1" >> ${DOCKER_FILE} - echo -e "RUN /source/"$@"" >> ${DOCKER_FILE} - fi - - if [ "$RUN_EXEC" = "true" ]; then - "$@" - fi -} - -finish() -{ - if [ "$RUN_EXEC" = "true" ]; then - return - fi - - local cmd="docker build -t ${DOCKER_IMAGE} -f ${DOCKER_FILE} ." - echo Running $cmd - echo Pwd $(pwd) - - cat ${DOCKER_FILE} - - $cmd -} - -reset_docker_build() -{ - BASE_WRITTEN=false -} - -check() -{ - local hash=$1 - local file=$2 - - if [ ! -e "$file" ]; then - return 1 - fi - - if [ "$hash" = dev ]; then - return 0 - fi - - CURRENT=$(sha1sum $file | awk '{print $1}') - - [ "$hash" = "$CURRENT" ] -} - -download() -{ - mkdir -p ${ARTIFACTS} - - local url=$2 - local file=${ARTIFACTS}/$(basename $2) - local hash=$1 - - if [ "$3" != "" ]; then - file=${ARTIFACTS}/$3 - fi - - if ! check $hash $file; then - echo Downloading $url to $file - curl -sL $url > $file - fi - - if ! check $hash $file; then - echo "ERROR: $file does not match checksum $hash, got $CURRENT" 1>&2 - return 1 - fi -} +ARTIFACTS=$(pwd)/assets +BUILD=$(pwd)/build +DIST=$(pwd)/dist diff --git a/scripts/build-images b/scripts/build-images deleted file mode 100755 index 0584a785..00000000 --- a/scripts/build-images +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -ex - -cd $(dirname $0)/.. - -source scripts/build-common -source scripts/version -ARTIFACTS=${ARTIFACTS:?"ARTIFACTS not set"} -VERSION=${VERSION:?"VERSION not set"} -BUILD=${BUILD:?"BUILD not set"} -DIST=${DIST:?"DIST not set"} - -if [ -x "$(which wrapdocker)" ] && ! docker info >/dev/null 2>&1; then - wrapdocker -fi - -CONTAINER_INITRDBASE=$(docker create rancher/os-initrdbase:${VERSION}) -cleanup_initrdbase() { - docker rm -v ${CONTAINER_INITRDBASE} -} -trap cleanup_initrdbase EXIT -docker cp ${CONTAINER_INITRDBASE}:/initrd ${BUILD} # copies files to ${BUILD}/initrd - -INITRD_DIR=${BUILD}/initrd - - -mkdir -p ${DIST}/artifacts - -CONTAINER_KERNEL=$(docker create rancher/os-kernel:${VERSION}) -cleanup_kernel() { - docker rm -v ${CONTAINER_KERNEL} -} -trap cleanup_kernel EXIT -docker cp ${CONTAINER_KERNEL}:/kernel ${BUILD} # copies files to ${BUILD}/kernel -mv ${BUILD}/kernel/vmlinuz ${DIST}/artifacts/ #kernel - - -mv ${BUILD}/kernel/lib ${INITRD_DIR} #initrd2-kernel: /lib - -cp -f ${ARTIFACTS}/docker* ${INITRD_DIR}/docker #initrd2-docker: /docker -chmod +x ${INITRD_DIR}/docker #initrd2-docker: /docker - -ln -sf bin/rancheros ./ros - -for i in $(./ros c images -i os-config.yml); do - docker pull ${i} -done -docker save $(./ros c images -i os-config.yml) > ${INITRD_DIR}/images.tar #initrd2-images: /images.tar diff --git a/scripts/ci b/scripts/ci index 9a556c69..834c3598 100755 --- a/scripts/ci +++ b/scripts/ci @@ -3,57 +3,10 @@ set -ex cd $(dirname $0)/.. -export DOCKER_IMAGE=${DOCKER_IMAGE:=rancher-os-build} -export DOCKER_BASE=rancher/dind:v0.5.0 - -source scripts/build-common -mkdir -p ${BUILD} - -DOCKER_FILE=$(pwd)/.dockerfile - -generate_images() -{ - if [ "$RUN_EXEC" = "true" ]; then - ./scripts/build-images - return - fi - - IMAGE_ID=$(docker images --no-trunc -q ${DOCKER_IMAGE}) - - if [ -e ${BUILD}/${IMAGE_ID} ]; then - DOCKER_BASE=$(<${BUILD}/${IMAGE_ID}) - else - echo Running: docker run -d --privileged ${DOCKER_IMAGE} /source/scripts/build-images - CID=$(docker run -d --privileged ${DOCKER_IMAGE} /source/scripts/build-images) - docker logs -f ${CID} & - trap "docker rm -fv ${CID}" exit - [ "$(docker wait $CID)" == 0 ] - DOCKER_BASE=$(docker commit $CID) - - echo ${DOCKER_BASE} > ${BUILD}/${IMAGE_ID} - fi -} - if [ -x "$(which wrapdocker)" ] && ! docker info >/dev/null 2>&1; then wrapdocker fi -run ./scripts/bootstrap -ARGS= -for i in $(ls -d * .* | sort -u | grep -Ev '(\.|\.\.|\.dockerfile|build|tmp|dist|.DS_Store|.git|.idea|.vagrant|scripts|bin|state)$'); do - if [ -d $i ]; then - run --assets $i - else - ARGS="${ARGS} $i" - fi -done -run --assets ./scripts/version --assets "${ARGS}" ./scripts/build -run --assets ./scripts/build-common --assets ./assets ./scripts/download -run --assets ./scripts/build-images -finish - -generate_images - -reset_docker_build -run ./scripts/package "$@" -finish +docker build -t ros-build-base -f Dockerfile.base . +docker build -t ros-build . +echo make package | docker run -v /var/run/docker.sock:/var/run/docker.sock --name=ros-build -i ros-build diff --git a/scripts/clean b/scripts/clean deleted file mode 100755 index 693f0cbc..00000000 --- a/scripts/clean +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cd $(dirname $0)/.. - -rm -rf build dist gopath .dockerfile Dockerfile diff --git a/scripts/copy-images b/scripts/copy-images new file mode 100755 index 00000000..26e7e520 --- /dev/null +++ b/scripts/copy-images @@ -0,0 +1,26 @@ +#!/bin/bash +set -ex + +cd $(dirname $0)/.. + +. scripts/build-common +. scripts/version + +VERSION=${VERSION:?"VERSION not set"} +BUILD=${BUILD:?"BUILD not set"} + + +CONTAINER_INITRDBASE=$(docker create rancher/os-initrdbase:${VERSION}) +cleanup_initrdbase() { + docker rm -v ${CONTAINER_INITRDBASE} +} +trap cleanup_initrdbase EXIT +docker cp ${CONTAINER_INITRDBASE}:/initrd ${BUILD} # copies files to ${BUILD}/initrd + + +CONTAINER_KERNEL=$(docker create rancher/os-kernel:${VERSION}) +cleanup_kernel() { + docker rm -v ${CONTAINER_KERNEL} +} +trap cleanup_kernel EXIT +docker cp ${CONTAINER_KERNEL}:/kernel ${BUILD} # copies files to ${BUILD}/kernel diff --git a/scripts/download b/scripts/download deleted file mode 100755 index e9f28184..00000000 --- a/scripts/download +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd $(dirname $0)/.. - -source scripts/build-common - -download 99652ec62c682041b67aa917d9b040d5541b4e78 https://github.com/rancher/docker/releases/download/v1.7.1-ros-1/docker-1.7.1 diff --git a/scripts/isolinux.cfg b/scripts/isolinux.cfg new file mode 100644 index 00000000..602e0a46 --- /dev/null +++ b/scripts/isolinux.cfg @@ -0,0 +1,5 @@ +default rancheros +label rancheros + kernel /boot/vmlinuz + initrd /boot/initrd + append quiet rancher.password=rancher rancher.state.autoformat=[/dev/sda,/dev/vda] diff --git a/scripts/package b/scripts/package deleted file mode 100755 index 8a52faa9..00000000 --- a/scripts/package +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -ex - -cd $(dirname $0)/.. - -source scripts/build-common - -cp bin/rancheros ${BUILD}/initrd/init -cp ./os-config.yml ${BUILD}/initrd/ - -cd ${BUILD}/initrd - -if [ "--dev" = "$1" ]; then - find | cpio -H newc -o > ${DIST}/artifacts/initrd - cp ${DIST}/artifacts/initrd ${DIST}/artifacts/initrd.none -else - find | cpio -H newc -o | lzma -c > ${DIST}/artifacts/initrd -fi - -CD=${BUILD}/cd - -mkdir -p ${CD}/boot/isolinux -cp ${DIST}/artifacts/vmlinuz ${CD}/boot -cp ${DIST}/artifacts/initrd ${CD}/boot -cp /usr/lib/syslinux/isolinux.bin ${CD}/boot/isolinux -cp /usr/lib/syslinux/linux.c32 ${CD}/boot/isolinux/ldlinux.c32 - -cat > ${CD}/boot/isolinux/isolinux.cfg << EOF -default rancheros -label rancheros - kernel /boot/vmlinuz - initrd /boot/initrd - append quiet rancher.password=rancher rancher.state.autoformat=[/dev/sda,/dev/vda] -EOF - -cd ${CD} -# Copied from boot2docker, thanks. -xorriso \ - -publisher "Rancher Labs, Inc." \ - -as mkisofs \ - -l -J -R -V "RancherOS" \ - -no-emul-boot -boot-load-size 4 -boot-info-table \ - -b boot/isolinux/isolinux.bin -c boot/isolinux/boot.cat \ - -isohybrid-mbr /usr/lib/syslinux/isohdpfx.bin \ - -o ${DIST}/artifacts/rancheros.iso $(pwd) - - -if [ -e ${DIST}/artficats/iso-checksums.txt ]; then - rm ${DIST}/artficats/iso-checksums.txt -fi - -for algorithm in 'sha256' 'md5'; do - for i in ${DIST}/artifacts/*iso; do - echo "${algorithm}: $(${algorithm}sum ${i})" >> ${DIST}/artifacts/iso-checksums.txt - done -done diff --git a/scripts/release b/scripts/release deleted file mode 100755 index fc945bad..00000000 --- a/scripts/release +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -if ! docker info >/dev/null 2>&1 && [ -x "$(which wrapdocker)" ]; then - wrapdocker -fi - -exec $(dirname $0)/../build.sh diff --git a/scripts/run b/scripts/run index 1d3ffeb3..50f28d40 100755 --- a/scripts/run +++ b/scripts/run @@ -18,6 +18,14 @@ INITRD_CURRENT=${BUILD}/initrd-current INITRD_TEST=${BUILD}/initrd.test USER_DATA=cloud-init/openstack/latest/user_data +# PREREQ: brew install coreutils +path() { + local UNAME=$(uname) + if [ "$UNAME" == "Darwin" ]; then greadlink -f "$1" + elif [ "$UNAME" == "Linux" ]; then readlink -f "$1"; + fi +} + while [ "$#" -gt 0 ]; do case $1 in --append)